Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-03-20 22:06:33 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

merged with upstream/master

Browse Source
This commit is contained in:
csoler 2021-11-11 17:50:11 +01:00
commit bef780e0c7
876 changed files with 18344 additions and 15328 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -15,3 +15,5 @@ Thumbs.db
*.pro.user
.kdev4
*.kdev4
!supportlibs/libsam3/Makefile

11
.gitlab-ci.yml
View File

@ -13,7 +13,6 @@ workflow:
- if: $CI_COMMIT_BRANCH
variables:
REPO_ARGS_CMD: '[ -n "$CI_MERGE_REQUEST_ID" ] && echo --build-arg REPO_URL=\"$CI_MERGE_REQUEST_SOURCE_PROJECT_URL\" --build-arg REPO_BRANCH=\"$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME\" || echo --build-arg REPO_URL=\"$CI_REPOSITORY_URL\" --build-arg REPO_BRANCH=\"$CI_COMMIT_BRANCH\"'
UBUNTU_TESTING_IMAGE_TAG: "$CI_REGISTRY_IMAGE/gitlabci_outputs/ubuntu_testing:$CI_COMMIT_SHA"
build-ubuntu-test-image:
@ -27,7 +26,7 @@ build-ubuntu-test-image:
- >
docker build
-t $UBUNTU_TESTING_IMAGE_TAG
$($REPO_ARGS_CMD)
$($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
--file $CI_PROJECT_DIR/build_scripts/GitlabCI/gitlabCI.Dockerfile
Dockercontext
- docker push $UBUNTU_TESTING_IMAGE_TAG
@ -74,7 +73,9 @@ build-android-arm-apk:
script:
- mkdir Dockercontext
- >
docker build -t retroshare:android_arm_latest $($REPO_ARGS_CMD)
docker build -t retroshare:android_arm_latest
$($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
--build-arg RS_SERVICE_QMAKE_EXTRA_OPTS="$RS_SERVICE_QMAKE_EXTRA_OPTS"
--file $CI_PROJECT_DIR/build_scripts/GitlabCI/Android.Dockerfile
Dockercontext
# see https://stackoverflow.com/a/59055906
@ -92,9 +93,11 @@ build-android-arm64-apk:
script:
- mkdir Dockercontext
- >
docker build -t retroshare:android_arm64_latest $($REPO_ARGS_CMD)
docker build -t retroshare:android_arm64_latest
$($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
--build-arg ANDROID_PLATFORM_VER=21
--build-arg ANDROID_NDK_ARCH=arm64
--build-arg RS_SERVICE_QMAKE_EXTRA_OPTS="$RS_SERVICE_QMAKE_EXTRA_OPTS"
--file $CI_PROJECT_DIR/build_scripts/GitlabCI/Android.Dockerfile
Dockercontext
- >

3
.gitmodules vendored
View File

@ -14,3 +14,6 @@
[submodule "supportlibs/rapidjson"]
path = supportlibs/rapidjson
url = https://github.com/Tencent/rapidjson.git
[submodule "supportlibs/libsam3"]
path = supportlibs/libsam3
url = https://github.com/i2p/libsam3.git

13
build_scripts/Android/Dockerfile
View File

@ -16,6 +16,11 @@
# docker login registry.gitlab.com
# docker push ${CI_IMAGE_NAME}
## To extract the generated APK easily you can run after the build complete
# docker cp \
# $(docker create --rm ${CI_IMAGE_NAME}):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk \
# /tmp/RetroShare_Android_Service.apk
FROM ubuntu:20.04
@ -80,12 +85,12 @@ WORKDIR /jsonapi-generator-build/
RUN qmake ../RetroShare/jsonapi-generator/src/ \
CONFIG+=no_retroshare_plugins \
CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
CONFIG+=rs_jsonapi && \
CONFIG+=rs_jsonapi CONFIG+=no_rs_sam3_libsam3 && \
make -j$(nproc)
#CONFIG+=no_keywords
RUN mkdir /retroshare-service-android-build
WORKDIR /retroshare-service-android-build
ARG RS_SERVICE_QMAKE_EXTRA_OPTS
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
-spec android-clang \
CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
@ -93,7 +98,9 @@ RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
CONFIG+=no_rs_service_terminal_login
CONFIG+=no_rs_service_terminal_login \
CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3 \
$RS_SERVICE_QMAKE_EXTRA_OPTS
RUN make -j$(nproc)
RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/androiddeployqt \

1
build_scripts/Android/README.asciidoc
View File

@ -81,6 +81,7 @@ JSONAPI_GENERATOR_EXE=Your_Path/jsonapi-generator/src/jsonapi-generator
NATIVE_LIBS_TOOLCHAIN_PATH=Your_Path/retroshare-android-16-arm/
CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password
CONFIG+=no_rs_service_terminal_login
CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3
-------------------------------------------------------------------------------
TIP: Some versions of QtCreator try to find the Android SDK in

7
build_scripts/GitlabCI/Android.Dockerfile
View File

@ -17,7 +17,7 @@ WORKDIR /jsonapi-generator-build/
RUN qmake ../RetroShare/jsonapi-generator/src/ \
CONFIG+=no_retroshare_plugins \
CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
CONFIG+=rs_jsonapi && \
CONFIG+=rs_jsonapi CONFIG+=no_rs_sam3_libsam3 && \
make -j$(nproc)
RUN rm -rf /retroshare-service-android-build ; mkdir /retroshare-service-android-build
@ -25,6 +25,7 @@ WORKDIR /retroshare-service-android-build
# ARG declared before FROM get wiped after it, so we need declaring it again
ARG ANDROID_NDK_ARCH=arm
ARG RS_SERVICE_QMAKE_EXTRA_OPTS
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
-spec android-clang \
CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
@ -32,7 +33,9 @@ RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
CONFIG+=no_rs_service_terminal_login
CONFIG+=no_rs_service_terminal_login \
CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3 \
$RS_SERVICE_QMAKE_EXTRA_OPTS
RUN make -j$(nproc)
RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/

9
build_scripts/GitlabCI/getRepoArgs.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/sh
[ -n "$CI_MERGE_REQUEST_ID" ] &&
echo \
--build-arg REPO_URL="$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" \
--build-arg REPO_BRANCH="$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ||
echo \
--build-arg REPO_URL="$CI_REPOSITORY_URL" \
--build-arg REPO_BRANCH="$CI_COMMIT_BRANCH"

2
build_scripts/OBS

@ -1 +1 @@
Subproject commit 984ed34b7f751bd407877603551ead49959f1901
Subproject commit b260e58346b1eec782bdf88a7e8f3c9d36fd3ecb

6
build_scripts/Windows-msys2/build/pack.bat
View File

@ -147,8 +147,10 @@ for /R "%RsDeployPath%" %%D in (*.dll, *.exe) do (
call :copy_dependencies "%%D" "%RsDeployPath%"
)
echo copy qss
xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
if exist "%SourcePath%\retroshare-gui\src\qss" (
echo copy qss
xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
)
echo copy stylesheets
xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite%

13
build_scripts/Windows-msys2/env/tools/prepare-msys2.bat vendored
View File

@ -25,8 +25,11 @@ if exist "%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\pacman.exe" (
)
)
set MSYS2Install=msys2-base-%MSYS2Architecture%-20200720.tar.xz
set MSYS2Url=http://sourceforge.net/projects/msys2/files/Base/%MSYS2Architecture%/%MSYS2Install%/download
if "%MSYS2Architecture%"=="i686" set MSYS2Version=20210705
if "%MSYS2Architecture%"=="x86_64" set MSYS2Version=20210725
set MSYS2Install=msys2-base-%MSYS2Architecture%-%MSYS2Version%.tar.xz
set MSYS2Url=https://repo.msys2.org/distrib/%MSYS2Architecture%/%MSYS2Install%
%cecho% info "Remove previous MSYS2 version"
call "%ToolsPath%\remove-dir.bat" "%EnvMSYS2Path%"
@ -41,12 +44,10 @@ if not exist "%EnvDownloadPath%\%MSYS2Install%" %cecho% error "Cannot download M
set MSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh
%cecho% info "Initialize MSYS2"
"%MSYS2SH%" -lc "pacman -Sy"
"%MSYS2SH%" -lc "pacman --noconfirm --needed -S bash pacman pacman-mirrors msys2-runtime"
"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
"%MSYS2SH%" -lc "pacman --noconfirm -Su"
call "%EnvMSYS2Path%\msys%MSYS2Base%\autorebase.bat"
call "%EnvRootPath%\update-msys2.bat"
call "%EnvRootPath%\update-msys2.bat"
:exit
endlocal

6
build_scripts/Windows-msys2/env/tools/root/update-msys2.bat vendored
View File

@ -8,11 +8,11 @@ if exist "%~dp0msys2\msys64" call :update 64
goto :EOF
:update
set MSYSSH=%~dp0msys2\msys%~1\usr\bin\sh
set MSYS2SH=%~dp0msys2\msys%~1\usr\bin\sh
echo Update MSYS2 %~1
"%MSYSSH%" -lc "pacman -Sy"
"%MSYSSH%" -lc "pacman --noconfirm -Su"
"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
"%MSYS2SH%" -lc "pacman --noconfirm -Su"
:exit
endlocal

2
build_scripts/Windows-msys2/installer/retroshare-Qt5.nsi
View File

@ -218,7 +218,7 @@ Section $(Section_Data) Section_Data
; Stylesheets
SetOutPath "$INSTDIR\qss"
File /r "${DEPLOYDIR}\qss\*.*"
File /nonfatal /r "${DEPLOYDIR}\qss\*.*"
SectionEnd
;Section $(Section_Link) Section_Link

6
build_scripts/Windows/build/pack.bat
View File

@ -132,8 +132,10 @@ if exist "%QtPath%\..\plugins\styles\qwindowsvistastyle.dll" (
copy "%QtPath%\..\plugins\imageformats\*.dll" "%RsDeployPath%\imageformats" %Quite%
del /Q "%RsDeployPath%\imageformats\*d?.dll" %Quite%
echo copy qss
xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
if exist "%SourcePath%\retroshare-gui\src\qss" (
echo copy qss
xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
)
echo copy stylesheets
xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite%

17
build_scripts/Windows/env/tools/prepare-msys2.bat vendored
View File

@ -25,8 +25,11 @@ if exist "%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\pacman.exe" (
)
)
set MSYS2Install=msys2-base-%MSYS2Architecture%-20190524.tar.xz
set MSYS2Url=http://sourceforge.net/projects/msys2/files/Base/%MSYS2Architecture%/%MSYS2Install%/download
if "%MSYS2Architecture%"=="i686" set MSYS2Version=20210705
if "%MSYS2Architecture%"=="x86_64" set MSYS2Version=20210725
set MSYS2Install=msys2-base-%MSYS2Architecture%-%MSYS2Version%.tar.xz
set MSYS2Url=https://repo.msys2.org/distrib/%MSYS2Architecture%/%MSYS2Install%
set CMakeInstall=cmake-3.19.0-win32-x86.zip
set CMakeUrl=https://github.com/Kitware/CMake/releases/download/v3.19.0/%CMakeInstall%
set CMakeUnpackPath=%EnvMSYS2Path%\msys%MSYS2Base%
@ -64,17 +67,11 @@ if "%FoundProfile%"=="0" (
set MSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh
%cecho% info "Update keyring"
"%MSYS2SH%" -lc "curl -O http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
"%MSYS2SH%" -lc "pacman --noconfirm -U msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
%cecho% info "Initialize MSYS2"
"%MSYS2SH%" -lc "pacman -Sy"
"%MSYS2SH%" -lc "pacman --noconfirm --needed -S bash pacman pacman-mirrors msys2-runtime"
"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
"%MSYS2SH%" -lc "pacman --noconfirm -Su"
call "%EnvMSYS2Path%\msys%MSYS2Base%\autorebase.bat"
call "%EnvRootPath%\update-msys2.bat"
call "%EnvRootPath%\update-msys2.bat"
:exit
endlocal

6
build_scripts/Windows/env/tools/root/update-msys2.bat vendored
View File

@ -8,11 +8,11 @@ if exist "%~dp0msys2\msys64" call :update 64
goto :EOF
:update
set MSYSSH=%~dp0msys2\msys%~1\usr\bin\sh
set MSYS2SH=%~dp0msys2\msys%~1\usr\bin\sh
echo Update MSYS2 %~1
"%MSYSSH%" -lc "pacman -Sy"
"%MSYSSH%" -lc "pacman --noconfirm -Su"
"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
"%MSYS2SH%" -lc "pacman --noconfirm -Su"
:exit
endlocal

2
build_scripts/Windows/installer/retroshare-Qt5.nsi
View File

@ -334,7 +334,7 @@ Section $(Section_Data) Section_Data
; Stylesheets
SetOutPath "$INSTDIR\qss"
File /r "${SOURCEDIR}\retroshare-gui\src\qss\*.*"
File /nonfatal /r "${SOURCEDIR}\retroshare-gui\src\qss\*.*"
SectionEnd
;Section $(Section_Link) Section_Link

191
libretroshare/src/deep_search/commonutils.cpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -18,54 +18,49 @@
* *
*******************************************************************************/
#include <algorithm>
#include <thread>
#include "deep_search/commonutils.hpp"
#include "util/stacktrace.h"
#include "util/rsdebug.h"
#include "util/rsthreads.h"
#include "util/rsdebuglevel0.h"
#ifndef XAPIAN_AT_LEAST
/// Added in Xapian 1.4.2.
#define XAPIAN_AT_LEAST(A,B,C) \
(XAPIAN_MAJOR_VERSION > (A) || \
(XAPIAN_MAJOR_VERSION == (A) && \
(XAPIAN_MINOR_VERSION > (B) || \
(XAPIAN_MINOR_VERSION == (B) && XAPIAN_REVISION >= (C)))))
#endif
namespace DeepSearch
{
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags, int blockSize )
{
try
{
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
new Xapian::WritableDatabase(path, flags, blockSize) );
return dbPtr;
}
catch(Xapian::DatabaseLockError)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
print_stacktrace();
}
catch(...)
{
RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
<< "deleting it might help without causing any harm: "
<< path << std::endl;
print_stacktrace();
}
return nullptr;
}
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags )
{
try
{
#if XAPIAN_AT_LEAST(1,3,2)
std::unique_ptr<Xapian::Database> dbPtr(
new Xapian::Database(path, flags) );
#else
std::unique_ptr<Xapian::Database> dbPtr(new Xapian::Database(path));
if(flags)
{
RS_WARN( "Xapian DB flags: ", flags, " ignored due to old Xapian "
"library version: ", XAPIAN_VERSION, " < 1.3.2" );
}
#endif
return dbPtr;
}
catch(Xapian::DatabaseOpeningError e)
catch(Xapian::DatabaseOpeningError& e)
{
RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg()
<< ", probably nothing has been indexed yet." << std::endl;
}
catch(Xapian::DatabaseLockError)
catch(Xapian::DatabaseLockError&)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
@ -90,4 +85,136 @@ std::string timetToXapianDate(const rstime_t& time)
return date;
}
StubbornWriteOpQueue::~StubbornWriteOpQueue()
{
auto fErr = flush(0);
if(fErr)
{
RS_FATAL( "Flush failed on destruction ", mOpStore.size(),
" operations irreparably lost ", fErr );
print_stacktrace();
}
}
void StubbornWriteOpQueue::push(write_op op)
{
RS_DBG4("");
{
std::unique_lock<std::mutex> lock(mQueueMutex);
mOpStore.push(op);
}
flush();
}
std::error_condition StubbornWriteOpQueue::flush(
rstime_t acceptDelay, rstime_t callTS )
{
RS_DBG4("");
{
// Return without attempt to open the database if the queue is empty
std::unique_lock<std::mutex> lock(mQueueMutex);
if(mOpStore.empty()) return std::error_condition();
}
std::unique_ptr<Xapian::WritableDatabase> dbPtr;
try
{
dbPtr = std::make_unique<Xapian::WritableDatabase>(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
}
catch(Xapian::DatabaseLockError)
{
if(acceptDelay)
{
rstime_t tNow = time(nullptr);
rstime_t maxRemaining = tNow - (callTS + acceptDelay);
if(maxRemaining > 0)
{
std::chrono::milliseconds interval(
std::max(rstime_t(50), maxRemaining*1000/5) );
RS_DBG3( "Cannot acquire database write lock, retrying in:",
interval.count(), "ms" );
RsThread::async([this, acceptDelay, callTS, interval]()
{
std::this_thread::sleep_for(interval);
flush(acceptDelay, callTS);
});
return std::error_condition();
}
else
{
RS_ERR(std::errc::timed_out, acceptDelay, callTS, tNow);
return std::errc::timed_out;
}
}
else return std::errc::resource_unavailable_try_again;
}
catch(...)
{
RS_ERR("Xapian DB ", mDbPath, " is apparently corrupted");
print_stacktrace();
return std::errc::io_error;
}
std::unique_lock<std::mutex> lock(mQueueMutex);
while(!mOpStore.empty())
{
auto op = mOpStore.front(); mOpStore.pop();
op(*dbPtr);
}
return std::error_condition();
}
std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc)
{
if(rsHtmlDoc.empty()) return rsHtmlDoc;
const bool isPlainMsg =
rsHtmlDoc[0] != '<' || rsHtmlDoc[rsHtmlDoc.size() - 1] != '>';
if(isPlainMsg) return rsHtmlDoc;
auto oSize = rsHtmlDoc.size();
auto bodyTagBegin(rsHtmlDoc.find("<body"));
if(bodyTagBegin >= oSize) return rsHtmlDoc;
auto bodyTagEnd(rsHtmlDoc.find(">", bodyTagBegin));
if(bodyTagEnd >= oSize) return rsHtmlDoc;
std::string retVal(rsHtmlDoc.substr(bodyTagEnd+1));
// strip also CSS inside <style></style>
oSize = retVal.size();
auto styleTagBegin(retVal.find("<style"));
if(styleTagBegin < oSize)
{
auto styleEnd(retVal.find("</style>", styleTagBegin));
if(styleEnd < oSize)
retVal.erase(styleTagBegin, 8+styleEnd-styleTagBegin);
}
std::string::size_type oPos;
std::string::size_type cPos;
int itCount = 0;
while((oPos = retVal.find("<")) < retVal.size())
{
if((cPos = retVal.find(">")) <= retVal.size())
retVal.erase(oPos, 1+cPos-oPos);
else break;
// Avoid infinite loop with crafty input
if(itCount > 1000)
{
RS_WARN( "Breaking stripping loop due to max allowed iterations ",
"rsHtmlDoc: ", rsHtmlDoc, " retVal: ", retVal );
break;
}
++itCount;
}
return retVal;
}
}

34
libretroshare/src/deep_search/commonutils.hpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -21,6 +21,9 @@
#include <xapian.h>
#include <memory>
#include <functional>
#include <queue>
#include <mutex>
#include "util/rstime.h"
@ -33,13 +36,34 @@
namespace DeepSearch
{
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags = 0, int blockSize = 0 );
typedef std::function<void(Xapian::WritableDatabase&)> write_op;
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags = 0 );
std::string timetToXapianDate(const rstime_t& time);
std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc);
struct StubbornWriteOpQueue
{
explicit StubbornWriteOpQueue(const std::string& dbPath):
mDbPath(dbPath) {}
~StubbornWriteOpQueue();
void push(write_op op);
std::error_condition flush(
rstime_t acceptDelay = 20, rstime_t callTS = time(nullptr) );
private:
std::queue<write_op> mOpStore;
rstime_t mLastFlush;
std::mutex mQueueMutex;
const std::string mDbPath;
};
}

83
libretroshare/src/deep_search/filesindex.cpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -18,47 +18,47 @@
* *
*******************************************************************************/
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebug.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
#include <utility>
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebuglevel1.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
/*static*/ std::multimap<int, DeepFilesIndex::IndexerFunType>
DeepFilesIndex::indexersRegister = {};
bool DeepFilesIndex::indexFile(
std::error_condition DeepFilesIndex::indexFile(
const std::string& path, const std::string& name,
const RsFileHash& hash )
{
auto dbPtr = DeepSearch::openWritableDatabase(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
if(!dbPtr) return false;
Xapian::WritableDatabase& db(*dbPtr);
const std::string hashString = hash.toStdString();
const std::string idTerm("Q" + hashString);
Xapian::Document oldDoc;
Xapian::PostingIterator pIt = db.postlist_begin(idTerm);
if( pIt != db.postlist_end(idTerm) )
auto db = DeepSearch::openReadOnlyDatabase(mDbPath);
if(db)
{
oldDoc = db.get_document(*pIt);
if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
RS_HUMAN_READABLE_VERSION &&
std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
indexersRegister.size() )
Xapian::Document oldDoc;
Xapian::PostingIterator pIt = db->postlist_begin(idTerm);
if( pIt != db->postlist_end(idTerm) )
{
/* Looks like this file has already been indexed by this RetroShare
* exact version, so we can skip it. If the version was different it
* made sense to reindex it as better indexers might be available
* since last time it was indexed */
Dbg3() << __PRETTY_FUNCTION__ << " skipping laready indexed file: "
<< hash << " " << name << std::endl;
return true;
oldDoc = db->get_document(*pIt);
if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
RS_HUMAN_READABLE_VERSION &&
std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
indexersRegister.size() )
{
/* Looks like this file has already been indexed by this
* RetroShare exact version, so we can skip it. If the version
* was different it made sense to reindex it as better indexers
* might be available since last time it was indexed */
RS_DBG3("skipping laready indexed file: ", hash, " ", name);
return std::error_condition();
}
}
db.reset(); // Release DB read lock ASAP
}
Xapian::Document doc;
@ -80,22 +80,21 @@ bool DeepFilesIndex::indexFile(
doc.add_value(
INDEXERS_COUNT_VALUENO,
std::to_string(indexersRegister.size()) );
db.replace_document(idTerm, doc);
return true;
mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); });
return std::error_condition();
}
bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
std::error_condition DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
{
Dbg3() << __PRETTY_FUNCTION__ << " removing file from index: "
<< hash << std::endl;
RS_DBG3(hash);
std::unique_ptr<Xapian::WritableDatabase> db =
DeepSearch::openWritableDatabase(mDbPath, Xapian::DB_CREATE_OR_OPEN);
if(!db) return false;
mWriteQueue.push([hash](Xapian::WritableDatabase& db)
{ db.delete_document("Q" + hash.toStdString()); });
db->delete_document("Q" + hash.toStdString());
return true;
return std::error_condition();
}
/*static*/ std::string DeepFilesIndex::dbDefaultPath()
@ -104,20 +103,20 @@ bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
/*static*/ bool DeepFilesIndex::registerIndexer(
int order, const DeepFilesIndex::IndexerFunType& indexerFun )
{
Dbg1() << __PRETTY_FUNCTION__ << " " << order << std::endl;
RS_DBG1(order);
indexersRegister.insert(std::make_pair(order, indexerFun));
return true;
}
uint32_t DeepFilesIndex::search(
std::error_condition DeepFilesIndex::search(
const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results, uint32_t maxResults )
{
results.clear();
auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath);
if(!dbPtr) return 0;
if(!dbPtr) return std::errc::bad_file_descriptor;
Xapian::Database& db(*dbPtr);
// Set up a QueryParser with a stemmer and suitable prefixes.
@ -151,7 +150,7 @@ uint32_t DeepFilesIndex::search(
results.push_back(s);
}
return static_cast<uint32_t>(results.size());
return std::error_condition();
}

23
libretroshare/src/deep_search/filesindex.hpp
View File

@ -1,8 +1,8 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
@ -19,9 +19,6 @@
*******************************************************************************/
#pragma once
#include "retroshare/rstypes.h"
#include "util/rsdebug.h"
#include <string>
#include <cstdint>
#include <vector>
@ -29,6 +26,9 @@
#include <map>
#include <functional>
#include "retroshare/rstypes.h"
#include "deep_search/commonutils.hpp"
struct DeepFilesSearchResult
{
DeepFilesSearchResult() : mWeight(0) {}
@ -41,7 +41,8 @@ struct DeepFilesSearchResult
class DeepFilesIndex
{
public:
DeepFilesIndex(const std::string& dbPath) : mDbPath(dbPath) {}
explicit DeepFilesIndex(const std::string& dbPath):
mDbPath(dbPath), mWriteQueue(dbPath) {}
/**
* @brief Search indexed files
@ -49,7 +50,7 @@ public:
* no limits
* @return search results count
*/
uint32_t search( const std::string& queryStr,
std::error_condition search( const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results,
uint32_t maxResults = 100 );
@ -57,7 +58,7 @@ public:
* @return false if file could not be indexed because of error or
* unsupported type, true otherwise.
*/
bool indexFile(
std::error_condition indexFile(
const std::string& path, const std::string& name,
const RsFileHash& hash );
@ -65,7 +66,7 @@ public:
* @brief Remove file entry from database
* @return false on error, true otherwise.
*/
bool removeFileFromIndex(const RsFileHash& hash);
std::error_condition removeFileFromIndex(const RsFileHash& hash);
static std::string dbDefaultPath();
@ -96,8 +97,8 @@ private:
const std::string mDbPath;
DeepSearch::StubbornWriteOpQueue mWriteQueue;
/** Storage for indexers function by order */
static std::multimap<int, IndexerFunType> indexersRegister;
RS_SET_CONTEXT_DEBUG_LEVEL(1)
};

208
libretroshare/src/deep_search/forumsindex.cpp Normal file
View File

@ -0,0 +1,208 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "deep_search/forumsindex.hpp"
#include "deep_search/commonutils.hpp"
#include "retroshare/rsinit.h"
#include "retroshare/rsgxsforums.h"
#include "util/rsdebuglevel4.h"
std::error_condition DeepForumsIndex::search(
const std::string& queryStr,
std::vector<DeepForumsSearchResult>& results, uint32_t maxResults )
{
results.clear();
std::unique_ptr<Xapian::Database> dbPtr(
DeepSearch::openReadOnlyDatabase(mDbPath) );
if(!dbPtr) return std::errc::bad_file_descriptor;
Xapian::Database& db(*dbPtr);
// Set up a QueryParser with a stemmer and suitable prefixes.
Xapian::QueryParser queryparser;
//queryparser.set_stemmer(Xapian::Stem("en"));
queryparser.set_stemming_strategy(queryparser.STEM_SOME);
// Start of prefix configuration.
//queryparser.add_prefix("title", "S");
//queryparser.add_prefix("description", "XD");
// End of prefix configuration.
// And parse the query.
Xapian::Query query = queryparser.parse_query(queryStr);
// Use an Enquire object on the database to run the query.
Xapian::Enquire enquire(db);
enquire.set_query(query);
Xapian::MSet mset = enquire.get_mset(
0, maxResults ? maxResults : db.get_doccount() );
for( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m )
{
const Xapian::Document& doc = m.get_document();
DeepForumsSearchResult s;
s.mUrl = doc.get_value(URL_VALUENO);
#if XAPIAN_AT_LEAST(1,3,5)
s.mSnippet = mset.snippet(doc.get_data());
#endif // XAPIAN_AT_LEAST(1,3,5)
results.push_back(s);
}
return std::error_condition();
}
/*static*/ std::string DeepForumsIndex::forumIndexId(const RsGxsGroupId& grpId)
{
RsUrl forumIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
forumIndexId.setQueryKV(
RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString() );
return forumIndexId.toString();
}
/*static*/ std::string DeepForumsIndex::postIndexId(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId )
{
RsUrl postIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL);
postIndexId.setQueryKV(RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString());
postIndexId.setQueryKV(RsGxsForums::FORUM_URL_MSG_ID_FIELD, msgId.toStdString());
return postIndexId.toString();
}
std::error_condition DeepForumsIndex::indexForumGroup(
const RsGxsForumGroup& forum )
{
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
// We make a document and tell the term generator to use this.
Xapian::Document doc;
termgenerator.set_document(doc);
// Index each field with a suitable prefix.
termgenerator.index_text(forum.mMeta.mGroupName, 1, "G");
termgenerator.index_text(
DeepSearch::timetToXapianDate(forum.mMeta.mPublishTs), 1, "D" );
termgenerator.index_text(forum.mDescription, 1, "XD");
// Index fields without prefixes for general search.
termgenerator.index_text(forum.mMeta.mGroupName);
termgenerator.increase_termpos();
termgenerator.index_text(forum.mDescription);
// store the RS link so we are able to retrive it on matching search
const std::string rsLink(forumIndexId(forum.mMeta.mGroupId));
doc.add_value(URL_VALUENO, rsLink);
/* Store some fields for display purposes. Retrieved later to provide the
* matching snippet on search */
doc.set_data(forum.mMeta.mGroupName + "\n" + forum.mDescription);
/* We use the identifier to ensure each object ends up in the database only
* once no matter how many times we run the indexer.
* "Q" prefix is a Xapian convention for unique id term. */
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm);
mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
}
std::error_condition DeepForumsIndex::removeForumFromIndex(
const RsGxsGroupId& grpId )
{
mWriteQueue.push([grpId](Xapian::WritableDatabase& db)
{ db.delete_document("Q" + forumIndexId(grpId)); });
return std::error_condition();
}
std::error_condition DeepForumsIndex::indexForumPost(const RsGxsForumMsg& post)
{
RS_DBG4(post);
const auto& groupId = post.mMeta.mGroupId;
const auto& msgId = post.mMeta.mMsgId;
if(groupId.isNull() || msgId.isNull())
{
RS_ERR("Got post with invalid id ", post);
print_stacktrace();
return std::errc::invalid_argument;
}
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
// We make a document and tell the term generator to use this.
Xapian::Document doc;
termgenerator.set_document(doc);
// Index each field with a suitable prefix.
termgenerator.index_text(post.mMeta.mMsgName, 1, "S");
termgenerator.index_text(
DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" );
// Avoid indexing RetroShare-gui HTML tags
const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg);
termgenerator.index_text(cleanMsg, 1, "XD" );
// Index fields without prefixes for general search.
termgenerator.index_text(post.mMeta.mMsgName);
termgenerator.increase_termpos();
termgenerator.index_text(cleanMsg);
// store the RS link so we are able to retrive it on matching search
const std::string rsLink(postIndexId(groupId, msgId));
doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes.
doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg);
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer.
const std::string idTerm("Q" + rsLink);
doc.add_boolean_term(idTerm);
mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db)
{ db.replace_document(idTerm, doc); } );
return std::error_condition();
}
std::error_condition DeepForumsIndex::removeForumPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId )
{
// "Q" prefix is a Xapian convention for unique id term.
std::string idTerm("Q" + postIndexId(grpId, msgId));
mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db)
{ db.delete_document(idTerm); } );
return std::error_condition();
}
/*static*/ std::string DeepForumsIndex::dbDefaultPath()
{ return RsAccounts::AccountDirectory() + "/deep_forum_index_xapian_db"; }

81
libretroshare/src/deep_search/forumsindex.hpp Normal file
View File

@ -0,0 +1,81 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
#include <system_error>
#include <vector>
#include <xapian.h>
#include "util/rstime.h"
#include "retroshare/rsgxsforums.h"
#include "retroshare/rsevents.h"
#include "deep_search/commonutils.hpp"
struct DeepForumsSearchResult
{
std::string mUrl;
double mWeight;
std::string mSnippet;
};
struct DeepForumsIndex
{
explicit DeepForumsIndex(const std::string& dbPath) :
mDbPath(dbPath), mWriteQueue(dbPath) {}
/**
* @brief Search indexed GXS groups and messages
* @param[in] maxResults maximum number of acceptable search results, 0 for
* no limits
* @return search results count
*/
std::error_condition search( const std::string& queryStr,
std::vector<DeepForumsSearchResult>& results,
uint32_t maxResults = 100 );
std::error_condition indexForumGroup(const RsGxsForumGroup& chan);
std::error_condition removeForumFromIndex(const RsGxsGroupId& grpId);
std::error_condition indexForumPost(const RsGxsForumMsg& post);
std::error_condition removeForumPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId );
static std::string dbDefaultPath();
private:
static std::string forumIndexId(const RsGxsGroupId& grpId);
static std::string postIndexId(
const RsGxsGroupId& grpId, const RsGxsMessageId& msgId );
enum : Xapian::valueno
{
/// Used to store retroshare url of indexed documents
URL_VALUENO,
/// @see Xapian::BAD_VALUENO
BAD_VALUENO = Xapian::BAD_VALUENO
};
const std::string mDbPath;
DeepSearch::StubbornWriteOpQueue mWriteQueue;
};

214
libretroshare/src/file_sharing/dir_hierarchy.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -29,6 +31,7 @@
#include "dir_hierarchy.h"
#include "filelist_io.h"
#include "file_sharing_defaults.h"
#include "util/cxx17retrocompat.h"
#ifdef RS_DEEP_FILES_INDEX
# include "deep_search/filesindex.hpp"
@ -70,7 +73,8 @@ InternalFileHierarchyStorage::InternalFileHierarchyStorage() : mRoot(0)
mTotalFiles = 0 ;
}
bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const
bool InternalFileHierarchyStorage::getDirHashFromIndex(
const DirectoryStorage::EntryIndex& index, RsFileHash& hash ) const
{
if(!checkIndex(index,FileStorageNode::TYPE_DIR))
return false ;
@ -79,6 +83,7 @@ bool InternalFileHierarchyStorage::getDirHashFromIndex(const DirectoryStorage::E
return true;
}
bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index)
{
std::map<RsFileHash,DirectoryStorage::EntryIndex>::iterator it = mDirHashes.find(hash) ;
@ -88,35 +93,39 @@ bool InternalFileHierarchyStorage::getIndexFromDirHash(const RsFileHash& hash,Di
index = it->second;
// make sure the hash actually points to some existing file. If not, remove it. This is a lazy update of dir hashes: when we need them, we check them.
if(!checkIndex(index, FileStorageNode::TYPE_DIR) || static_cast<DirEntry*>(mNodes[index])->dir_hash != hash)
{
std::cerr << "(II) removing non existing hash from dir hash list: " << hash << std::endl;
mDirHashes.erase(it) ;
return false ;
}
/* make sure the hash actually points to some existing directory. If not,
* remove it. This is an opportunistic update of dir hashes: when we need
* them, we check them. */
if( !checkIndex(index, FileStorageNode::TYPE_DIR) ||
static_cast<DirEntry*>(mNodes[index])->dir_hash != hash )
{
RS_INFO("removing non existing dir hash: ", hash, " from dir hash list");
mDirHashes.erase(it);
return false;
}
return true;
}
bool InternalFileHierarchyStorage::getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index)
bool InternalFileHierarchyStorage::getIndexFromFileHash(
const RsFileHash& hash, DirectoryStorage::EntryIndex& index )
{
std::map<RsFileHash,DirectoryStorage::EntryIndex>::iterator it = mFileHashes.find(hash) ;
auto it = std::as_const(mFileHashes).find(hash);
if(it == mFileHashes.end()) return false;
if(it == mFileHashes.end())
return false;
index = it->second;
index = it->second;
/* make sure the hash actually points to some existing file. If not, remove
* it. This is an opportunistic update of file hashes: when we need them,
* we check them. */
if( !checkIndex(it->second, FileStorageNode::TYPE_FILE) ||
static_cast<FileEntry*>(mNodes[index])->file_hash != hash )
{
RS_INFO("removing non existing file hash: ", hash, " from file hash list");
mFileHashes.erase(it);
return false;
}
// make sure the hash actually points to some existing file. If not, remove it. This is a lazy update of file hashes: when we need them, we check them.
if(!checkIndex(it->second, FileStorageNode::TYPE_FILE) || static_cast<FileEntry*>(mNodes[index])->file_hash != hash)
{
std::cerr << "(II) removing non existing hash from file hash list: " << hash << std::endl;
mFileHashes.erase(it) ;
return false ;
}
return true;
return true;
}
bool InternalFileHierarchyStorage::getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const
@ -155,10 +164,13 @@ bool InternalFileHierarchyStorage::isIndexValid(DirectoryStorage::EntryIndex e)
return e < mNodes.size() && mNodes[e] != NULL ;
}
bool InternalFileHierarchyStorage::updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx, const std::set<std::string>& subdirs, const RsFileHash& random_hash_seed)
bool InternalFileHierarchyStorage::updateSubDirectoryList(
const DirectoryStorage::EntryIndex& indx,
const std::set<std::string>& subdirs,
const RsFileHash& random_hash_seed )
{
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
return false;
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
return false;
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
@ -284,10 +296,17 @@ bool InternalFileHierarchyStorage::checkIndex(DirectoryStorage::EntryIndex indx,
return true;
}
bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,DirectoryStorage::FileTS>& subfiles,std::map<std::string,DirectoryStorage::FileTS>& new_files)
bool InternalFileHierarchyStorage::updateSubFilesList(
const DirectoryStorage::EntryIndex& indx,
const std::map<std::string,DirectoryStorage::FileTS>& subfiles,
std::map<std::string,DirectoryStorage::FileTS>& new_files )
{
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
return false;
if(!checkIndex(indx, FileStorageNode::TYPE_DIR))
{
RS_ERR("indx: ", indx, std::errc::not_a_directory);
print_stacktrace();
return false;
}
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
new_files = subfiles ;
@ -312,9 +331,11 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En
continue;
}
if(it->second.modtime != f.file_modtime || it->second.size != f.file_size) // file is newer and/or has different size
// file is newer and/or has different size
if(it->second.modtime != f.file_modtime || it->second.size != f.file_size)
{
f.file_hash.clear(); // hash needs recomputing
// hash needs recomputing
f.file_hash.clear();
f.file_modtime = it->second.modtime;
f.file_size = it->second.size;
@ -342,13 +363,16 @@ bool InternalFileHierarchyStorage::updateSubFilesList(const DirectoryStorage::En
}
return true;
}
bool InternalFileHierarchyStorage::updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash)
bool InternalFileHierarchyStorage::updateHash(
const DirectoryStorage::EntryIndex& file_index, const RsFileHash& hash )
{
if(!checkIndex(file_index,FileStorageNode::TYPE_FILE))
{
std::cerr << "[directory storage] (EE) cannot update file at index " << file_index << ". Not a valid index, or not a file." << std::endl;
return false;
}
if(!checkIndex(file_index, FileStorageNode::TYPE_FILE))
{
RS_ERR( "Cannot update file at index ", file_index,
". Not a valid index, or not a file." );
print_stacktrace();
return false;
}
#ifdef DEBUG_DIRECTORY_STORAGE
std::cerr << "[directory storage] updating hash at index " << file_index << ", hash=" << hash << std::endl;
#endif
@ -436,14 +460,20 @@ DirectoryStorage::EntryIndex InternalFileHierarchyStorage::allocateNewIndex()
return mNodes.size()-1 ;
}
bool InternalFileHierarchyStorage::updateDirEntry(const DirectoryStorage::EntryIndex& indx,const std::string& dir_name,rstime_t most_recent_time,rstime_t dir_modtime,const std::vector<RsFileHash>& subdirs_hash,const std::vector<FileEntry>& subfiles_array)
bool InternalFileHierarchyStorage::updateDirEntry(
const DirectoryStorage::EntryIndex& indx, const std::string& dir_name,
rstime_t most_recent_time, rstime_t dir_modtime,
const std::vector<RsFileHash>& subdirs_hash,
const std::vector<FileEntry>& subfiles_array )
{
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
{
std::cerr << "[directory storage] (EE) cannot update dir at index " << indx << ". Not a valid index, or not an existing dir." << std::endl;
return false;
}
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx])) ;
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
{
RS_ERR( "cannot update dir at index ", indx, ". Not a valid index, or "
"not an existing dir." );
return false;
}
DirEntry& d(*static_cast<DirEntry*>(mNodes[indx]));
#ifdef DEBUG_DIRECTORY_STORAGE
std::cerr << "Updating dir entry: name=\"" << dir_name << "\", most_recent_time=" << most_recent_time << ", modtime=" << dir_modtime << std::endl;
@ -703,14 +733,14 @@ const InternalFileHierarchyStorage::FileStorageNode *InternalFileHierarchyStorag
return NULL ;
}
const InternalFileHierarchyStorage::DirEntry *InternalFileHierarchyStorage::getDirEntry(DirectoryStorage::EntryIndex indx) const
const InternalFileHierarchyStorage::DirEntry*
InternalFileHierarchyStorage::getDirEntry(DirectoryStorage::EntryIndex indx) const
{
if(!checkIndex(indx,FileStorageNode::TYPE_DIR))
return NULL ;
return static_cast<DirEntry*>(mNodes[indx]) ;
if(!checkIndex(indx,FileStorageNode::TYPE_DIR)) return nullptr;
return static_cast<DirEntry*>(mNodes[indx]);
}
const InternalFileHierarchyStorage::FileEntry *InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const
const InternalFileHierarchyStorage::FileEntry*
InternalFileHierarchyStorage::getFileEntry(DirectoryStorage::EntryIndex indx) const
{
if(!checkIndex(indx,FileStorageNode::TYPE_FILE))
return NULL ;
@ -754,7 +784,10 @@ bool InternalFileHierarchyStorage::searchHash(const RsFileHash& hash,DirectorySt
class DirectoryStorageExprFileEntry: public RsRegularExpression::ExpFileEntry
{
public:
DirectoryStorageExprFileEntry(const InternalFileHierarchyStorage::FileEntry& fe,const InternalFileHierarchyStorage::DirEntry& parent) : mFe(fe),mDe(parent) {}
DirectoryStorageExprFileEntry(
const InternalFileHierarchyStorage::FileEntry& fe,
const InternalFileHierarchyStorage::DirEntry& parent ) :
mFe(fe), mDe(parent) {}
inline virtual const std::string& file_name() const { return mFe.file_name ; }
inline virtual uint64_t file_size() const { return mFe.file_size ; }
@ -768,14 +801,18 @@ private:
const InternalFileHierarchyStorage::DirEntry& mDe ;
};
int InternalFileHierarchyStorage::searchBoolExp(RsRegularExpression::Expression * exp, std::list<DirectoryStorage::EntryIndex> &results) const
int InternalFileHierarchyStorage::searchBoolExp(
RsRegularExpression::Expression* exp,
std::list<DirectoryStorage::EntryIndex>& results ) const
{
for(std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it)
if(mNodes[it->second] != NULL && exp->eval(
DirectoryStorageExprFileEntry(*static_cast<const FileEntry*>(mNodes[it->second]),
*static_cast<const DirEntry*>(mNodes[mNodes[it->second]->parent_index])
)))
results.push_back(it->second);
for(auto& it: std::as_const(mFileHashes))
if(mNodes[it.second])
if(exp->eval(
DirectoryStorageExprFileEntry(
*static_cast<const FileEntry*>(mNodes[it.second]),
*static_cast<const DirEntry*>(mNodes[mNodes[it.second]->parent_index])
) ))
results.push_back(it.second);
return 0;
}
@ -784,27 +821,43 @@ int InternalFileHierarchyStorage::searchTerms(
const std::list<std::string>& terms,
std::list<DirectoryStorage::EntryIndex>& results ) const
{
// most entries are likely to be files, so we could do a linear search over the entries tab.
// instead we go through the table of hashes.
/* most entries are likely to be files, so we could do a linear search over
* the entries tab. Instead we go through the table of hashes.*/
for(std::map<RsFileHash,DirectoryStorage::EntryIndex>::const_iterator it(mFileHashes.begin());it!=mFileHashes.end();++it)
if(mNodes[it->second] != NULL)
{
const std::string &str1 = static_cast<FileEntry*>(mNodes[it->second])->file_name;
for(auto& it : std::as_const(mFileHashes))
{
// node may be null for some hash waiting to be deleted
if(mNodes[it.second])
{
rs_view_ptr<FileEntry> tFileEntry =
static_cast<FileEntry*>(mNodes[it.second]);
for(std::list<std::string>::const_iterator iter(terms.begin()); iter != terms.end(); ++iter)
{
/* always ignore case */
const std::string &str2 = (*iter);
/* Most file will just have file name stored, but single file shared
* without a shared dir will contain full path instead of just the
* name, so purify it to perform the search */
std::string tFilename = tFileEntry->file_name;
if(tFileEntry->file_name.find("/") != std::string::npos)
{
std::string _tParentDir;
RsDirUtil::splitDirFromFile(
tFileEntry->file_name, _tParentDir, tFilename );
}
if(str1.end() != std::search( str1.begin(), str1.end(), str2.begin(), str2.end(), RsRegularExpression::CompareCharIC() ))
{
results.push_back(it->second);
break;
}
}
}
return 0 ;
for(auto& termIt : std::as_const(terms))
{
/* always ignore case */
if(tFilename.end() != std::search(
tFilename.begin(), tFilename.end(),
termIt.begin(), termIt.end(),
RsRegularExpression::CompareCharIC() ))
{
results.push_back(it.second);
break;
}
}
}
}
return 0;
}
bool InternalFileHierarchyStorage::check(std::string& error_string) // checks consistency of storage.
@ -958,8 +1011,9 @@ void InternalFileHierarchyStorage::recursPrint(int depth,DirectoryStorage::Entry
bool InternalFileHierarchyStorage::nodeAccessError(const std::string& s)
{
std::cerr << "(EE) InternalDirectoryStructure: ERROR: " << s << std::endl;
return false ;
RS_ERR(s);
print_stacktrace();
return false;
}
// Removes the given subdirectory from the parent node and all its pendign subdirs and files.

369
libretroshare/src/file_sharing/directory_storage.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -30,6 +32,7 @@
#include "directory_storage.h"
#include "dir_hierarchy.h"
#include "filelist_io.h"
#include "util/cxx17retrocompat.h"
#ifdef RS_DEEP_FILES_INDEX
# include "deep_search/filesindex.hpp"
@ -67,8 +70,11 @@ DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++()
return *this;
}
DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const { return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; }
DirectoryStorage::EntryIndex DirectoryStorage::DirIterator ::operator*() const { return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; }
DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const
{ return mStorage->getSubFileIndex(mParentIndex, mFileTabIndex); }
DirectoryStorage::EntryIndex DirectoryStorage::DirIterator::operator*() const
{ return mStorage->getSubDirIndex(mParentIndex, mDirTabIndex); }
DirectoryStorage::FileIterator::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
DirectoryStorage::DirIterator ::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
@ -139,7 +145,9 @@ bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx, const std:
mChanged = true ;
return res ;
}
bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map<std::string,FileTS>& subfiles,std::map<std::string,FileTS>& new_files)
bool DirectoryStorage::updateSubFilesList(
const EntryIndex& indx, const std::map<std::string,FileTS>& subfiles,
std::map<std::string,FileTS>& new_files )
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ;
@ -348,7 +356,8 @@ int LocalDirectoryStorage::searchHash(const RsFileHash& hash, RsFileHash& real_h
return false ;
}
void LocalDirectoryStorage::setSharedDirectoryList(const std::list<SharedDirInfo>& lst)
void LocalDirectoryStorage::setSharedDirectoryList(
const std::list<SharedDirInfo>& lst )
{
std::set<std::string> dirs_with_new_virtualname ;
bool dirs_with_changed_flags = false ;
@ -379,7 +388,9 @@ void LocalDirectoryStorage::setSharedDirectoryList(const std::list<SharedDirInfo
virtual_names.insert(candidate_virtual_name) ;
}
// now for each member of the processed list, check if it is an existing shared directory that has been changed. If so, we need to update the dir TS of that directory
/* now for each member of the processed list, check if it is an existing
* shared directory that has been changed. If so, we need to update the
* dir TS of that directory */
std::map<std::string,SharedDirInfo> new_dirs ;
@ -529,7 +540,7 @@ bool LocalDirectoryStorage::updateHash(
#endif
ret = (!update_internal_hierarchy) ||
mFileHierarchy->updateHash(index,hash);
mFileHierarchy->updateHash(index, hash);
} // RS_STACK_MUTEX(mDirStorageMtx);
#ifdef RS_DEEP_FILES_INDEX
@ -538,7 +549,7 @@ bool LocalDirectoryStorage::updateHash(
fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH )
{
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash);
ret &= !dfi.indexFile(fInfo.path, fInfo.fname, hash);
}
#endif // def RS_DEEP_FILES_INDEX
@ -617,48 +628,78 @@ bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,Fil
return locked_getFileSharingPermissions(indx,flags,parent_groups) ;
}
bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags& flags, std::list<RsNodeGroupId> &parent_groups)
bool LocalDirectoryStorage::locked_getFileSharingPermissions(
const EntryIndex& indx, FileStorageFlags& flags,
std::list<RsNodeGroupId>& parent_groups )
{
flags.clear() ;
parent_groups.clear();
flags.clear();
parent_groups.clear();
std::string base_dir;
/* We got a request for root directory no need to do anything more after
* clearing outputs */
if(!indx) return true;
const InternalFileHierarchyStorage::FileStorageNode *n = mFileHierarchy->getNode(indx) ;
using FileStorageNode = InternalFileHierarchyStorage::FileStorageNode;
using EntryIndex = DirectoryStorage::EntryIndex;
if(n == NULL)
return false ;
rs_view_ptr<const FileStorageNode> n = mFileHierarchy->getNode(indx);
if(!n)
{
RS_ERR("Node for index: ", indx, "not found");
print_stacktrace();
return false;
}
for(DirectoryStorage::EntryIndex i=((n->type()==InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)?((intptr_t)n->parent_index):indx);;)
{
const InternalFileHierarchyStorage::DirEntry *e = mFileHierarchy->getDirEntry(i) ;
// Climb down node tree up to root + 1
EntryIndex curIndex = indx;
while (n->parent_index)
{
curIndex = n->parent_index;
n = mFileHierarchy->getNode(curIndex);
}
if(e == NULL)
break ;
// Retrieve base name
std::string tBaseName;
switch (n->type())
{
// Handle single file shared case
case InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE:
tBaseName = mFileHierarchy->getFileEntry(curIndex)->file_name;
break;
// Handle shared directory case
case InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR:
tBaseName = mFileHierarchy->getDirEntry(curIndex)->dir_name;
break;
default:
RS_ERR("Got unhandled node type: ", n->type());
print_stacktrace();
return false;
}
if(e->parent_index == 0)
{
base_dir = e->dir_name ;
break ;
}
i = e->parent_index ;
}
// Use base name to retrieve sharing permissions
if(!tBaseName.empty())
{
auto it = std::as_const(mLocalDirs).find(tBaseName);
if(!base_dir.empty())
{
std::map<std::string,SharedDirInfo>::const_iterator it = mLocalDirs.find(base_dir) ;
if(it == mLocalDirs.end())
{
RS_ERR( "base name \"", tBaseName,
"\" for index: ", indx, " not found in shared dir list." );
print_stacktrace();
return false;
}
if(it == mLocalDirs.end())
{
std::cerr << "(II) base directory \"" << base_dir << "\" not found in shared dir list." << std::endl;
return false ;
}
flags = it->second.shareflags;
parent_groups = it->second.parent_groups;
}
else
{
RS_ERR("base name for indx: ", indx, " is empty");
print_stacktrace();
return false;
}
flags = it->second.shareflags;
parent_groups = it->second.parent_groups;
}
return true;
return true;
}
std::string LocalDirectoryStorage::locked_getVirtualDirName(EntryIndex indx) const
@ -705,134 +746,214 @@ std::string LocalDirectoryStorage::locked_getVirtualPath(EntryIndex indx) const
return it->second.virtualname + "/" + res;
}
bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id)
bool LocalDirectoryStorage::serialiseDirEntry(
const EntryIndex& indx, RsTlvBinaryData& bindata,
const RsPeerId& client_id )
{
RS_STACK_MUTEX(mDirStorageMtx) ;
RS_STACK_MUTEX(mDirStorageMtx);
const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx);
const InternalFileHierarchyStorage::DirEntry* dir =
mFileHierarchy->getDirEntry(indx);
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Serialising Dir entry " << std::hex << indx << " for client id " << client_id << std::endl;
#endif
if(dir == NULL)
{
std::cerr << "(EE) serialiseDirEntry: ERROR. Cannot find entry " << (void*)(intptr_t)indx << std::endl;
return false;
}
// compute list of allowed subdirs
std::vector<RsFileHash> allowed_subdirs ;
FileStorageFlags node_flags ;
std::list<RsNodeGroupId> node_groups ;
if(!dir)
{
RS_ERR("Cannot find entry ", indx);
return false;
}
// for each subdir, compute the node flags and groups, then ask rsPeers to compute the mask that result from these flags for the particular peer supplied in parameter
// compute list of allowed subdirs
std::vector<RsFileHash> allowed_subdirs;
FileStorageFlags node_flags;
std::list<RsNodeGroupId> node_groups;
/* for each subdir, compute the node flags and groups, then ask rsPeers to
* compute the mask that result from these flags for the particular peer
* supplied in parameter */
for(uint32_t i=0;i<dir->subdirs.size();++i)
if(indx != 0 || (
locked_getFileSharingPermissions(
dir->subdirs[i], node_flags, node_groups ) &&
( rsPeers->computePeerPermissionFlags(
client_id, node_flags, node_groups ) &
RS_FILE_HINTS_BROWSABLE ) ))
{
RsFileHash hash;
if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash))
{
RS_ERR( "Cannot get hash from subdir index: ",
dir->subdirs[i], ". Weird bug." );
print_stacktrace();
return false;
}
allowed_subdirs.push_back(hash);
for(uint32_t i=0;i<dir->subdirs.size();++i)
if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
{
RsFileHash hash ;
if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash))
{
std::cerr << "(EE) Cannot get hash from subdir index " << dir->subdirs[i] << ". Weird bug." << std::endl ;
return false;
}
allowed_subdirs.push_back(hash) ;
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << " pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << std::endl;
#endif
}
}
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
else
std::cerr << " not pushing subdir " << hash << ", array position=" << i << " indx=" << dir->subdirs[i] << ": permission denied for this peer." << std::endl;
#endif
// now count the files that do not have a null hash (meaning the hash has indeed been computed)
/* now count the files that do not have a null hash (meaning the hash has
* indeed been computed), also in case files are shared singularly (without
* a shared directory) so they are child of root check browsability
* permission */
uint32_t allowed_subfiles = 0;
for(uint32_t i=0; i<dir->subfiles.size(); ++i)
{
const InternalFileHierarchyStorage::FileEntry* file =
mFileHierarchy->getFileEntry(dir->subfiles[i]);
if(file != nullptr && !file->file_hash.isNull()
&& ( indx !=0 || (
locked_getFileSharingPermissions(
dir->subfiles[i], node_flags, node_groups ) &&
rsPeers->computePeerPermissionFlags(
client_id, node_flags, node_groups ) &
RS_FILE_HINTS_BROWSABLE ) ))
allowed_subfiles++;
}
uint32_t allowed_subfiles = 0 ;
unsigned char* section_data = (unsigned char *)
rs_malloc(FL_BASE_TMP_SECTION_SIZE);
if(!section_data) return false;
for(uint32_t i=0;i<dir->subfiles.size();++i)
{
const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ;
if(file != NULL && !file->file_hash.isNull())
allowed_subfiles++ ;
}
uint32_t section_size = FL_BASE_TMP_SECTION_SIZE;
uint32_t section_offset = 0;
unsigned char *section_data = (unsigned char *)rs_malloc(FL_BASE_TMP_SECTION_SIZE) ;
/* we need to send:
* - the name of the directory, its TS
* - the index entry for each subdir (the updte TS are exchanged at a
* higher level)
* - the file info for each subfile */
if(!section_data)
return false ;
std::string virtual_dir_name = locked_getVirtualDirName(indx);
uint32_t section_size = FL_BASE_TMP_SECTION_SIZE;
uint32_t section_offset = 0;
/* Manual serialization AGAIN! This is terrible and should be ported to the
* new serialization system ASAP! */
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_DIR_NAME, virtual_dir_name ))
{ free(section_data); return false; }
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_RECURS_MODIF_TS,
(uint32_t)dir->dir_most_recent_time ))
{ free(section_data); return false; }
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_MODIF_TS, (uint32_t)dir->dir_modtime ))
{ free(section_data); return false;}
// we need to send:
// - the name of the directory, its TS
// - the index entry for each subdir (the updte TS are exchanged at a higher level)
// - the file info for each subfile
//
std::string virtual_dir_name = locked_getVirtualDirName(indx) ;
// serialise number of subdirs and number of subfiles
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_RAW_NUMBER, (uint32_t)allowed_subdirs.size() ))
{ free(section_data); return false; }
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_RAW_NUMBER, (uint32_t)allowed_subfiles ))
{ free(section_data); return false; }
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,virtual_dir_name )) { free(section_data); return false ;}
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->dir_most_recent_time)) { free(section_data); return false ;}
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) { free(section_data); return false ;}
// serialise subdirs entry indexes
for(uint32_t i=0; i<allowed_subdirs.size(); ++i)
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_ENTRY_INDEX, allowed_subdirs[i] ))
{ free(section_data); return false; }
// serialise number of subdirs and number of subfiles
// serialise directory subfiles, with info for each of them
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subdirs.size() )) { free(section_data); return false ;}
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subfiles )) { free(section_data); return false ;}
unsigned char* file_section_data =
(unsigned char *) rs_malloc(FL_BASE_TMP_SECTION_SIZE);
if(!file_section_data)
{
free(section_data);
return false;
}
// serialise subdirs entry indexes
uint32_t file_section_size = FL_BASE_TMP_SECTION_SIZE;
for(uint32_t i=0;i<allowed_subdirs.size();++i)
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,allowed_subdirs[i] )) { free(section_data); return false ;}
for(uint32_t i=0; i<dir->subfiles.size(); ++i)
{
uint32_t file_section_offset = 0;
// serialise directory subfiles, with info for each of them
const InternalFileHierarchyStorage::FileEntry* file =
mFileHierarchy->getFileEntry(dir->subfiles[i]);
unsigned char *file_section_data = (unsigned char *)rs_malloc(FL_BASE_TMP_SECTION_SIZE) ;
if(file == nullptr || file->file_hash.isNull())
{
RS_INFO( "skipping unhashed or Null file entry ",
dir->subfiles[i], " to get/send file info." );
continue;
}
if(!file_section_data)
{
free(section_data);
return false ;
}
if(indx == 0)
{
if(!locked_getFileSharingPermissions(
dir->subfiles[i], node_flags, node_groups ))
{
RS_ERR( "Failure getting sharing permission for single file: ",
dir->subfiles[i] );
print_stacktrace();
continue;
}
uint32_t file_section_size = FL_BASE_TMP_SECTION_SIZE ;
if(!( rsPeers->computePeerPermissionFlags(
client_id, node_flags, node_groups ) &
RS_FILE_HINTS_BROWSABLE ))
{
RS_INFO( "Skipping single file shared without browse "
"permission" );
continue;
}
}
for(uint32_t i=0;i<dir->subfiles.size();++i)
{
uint32_t file_section_offset = 0 ;
if(!FileListIO::writeField(
file_section_data, file_section_size, file_section_offset,
FILE_LIST_IO_TAG_FILE_NAME, file->file_name ))
{ free(section_data); free(file_section_data); return false; }
if(!FileListIO::writeField(
file_section_data, file_section_size, file_section_offset,
FILE_LIST_IO_TAG_FILE_SIZE, file->file_size ))
{ free(section_data); free(file_section_data); return false; }
if(!FileListIO::writeField(
file_section_data, file_section_size, file_section_offset,
FILE_LIST_IO_TAG_FILE_SHA1_HASH, file->file_hash ))
{ free(section_data); free(file_section_data); return false; }
if(!FileListIO::writeField(
file_section_data, file_section_size, file_section_offset,
FILE_LIST_IO_TAG_MODIF_TS, (uint32_t)file->file_modtime ))
{ free(section_data); free(file_section_data); return false; }
const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ;
if(file == NULL || file->file_hash.isNull())
{
std::cerr << "(II) skipping unhashed or Null file entry " << dir->subfiles[i] << " to get/send file info." << std::endl;
continue ;
}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file->file_name )) { free(section_data);free(file_section_data);return false ;}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file->file_size )) { free(section_data);free(file_section_data);return false ;}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash )) { free(section_data);free(file_section_data);return false ;}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)file->file_modtime)) { free(section_data);free(file_section_data);return false ;}
// now write the whole string into a single section in the file
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_offset)) { free(section_data); free(file_section_data);return false ;}
// now write the whole string into a single section in the file
if(!FileListIO::writeField(
section_data, section_size, section_offset,
FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,
file_section_data, file_section_offset ))
{ free(section_data); free(file_section_data); return false; }
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << " pushing subfile " << file->hash << ", array position=" << i << " indx=" << dir->subfiles[i] << std::endl;
#endif
}
free(file_section_data) ;
}
free(file_section_data);
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl;
#endif
bindata.bin_data = realloc(section_data,section_offset) ; // This discards the possibly unused trailing bytes in the end of section_data
bindata.bin_len = section_offset ;
// Discards the possibly unused trailing bytes in the end of section_data
bindata.bin_data = realloc(section_data,section_offset);
bindata.bin_len = section_offset;
return true ;
return true;
}

356
libretroshare/src/file_sharing/directory_updater.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2016 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,13 +21,15 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
******************************************************************************/
#include "util/cxx17retrocompat.h"
#include "util/folderiterator.h"
#include "util/rstime.h"
#include "rsserver/p3face.h"
#include "directory_storage.h"
#include "directory_updater.h"
#include "file_sharing_defaults.h"
#include "util/rsdebuglevel3.h"
//#define DEBUG_LOCAL_DIR_UPDATER 1
@ -121,185 +125,238 @@ void LocalDirectoryUpdater::forceUpdate(bool add_safe_delay)
bool LocalDirectoryUpdater::sweepSharedDirectories(bool& some_files_not_ready)
{
if(mHashSalt.isNull())
{
std::cerr << "(EE) no salt value in LocalDirectoryUpdater. Is that a bug?" << std::endl;
return false;
}
if(mHashSalt.isNull())
{
RS_ERR("no salt value in LocalDirectoryUpdater");
print_stacktrace();
return false;
}
mIsChecking = true ;
mIsChecking = true;
RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] LocalDirectoryUpdater::sweep()" << std::endl;
#endif
RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
// recursive update algorithm works that way:
// - the external loop starts on the shared directory list and goes through sub-directories
// - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure.
// - the information that is costly to compute (the hash) is store externally into a separate structure.
// - doing so, changing directory names or moving files between directories does not cause a re-hash of the content.
//
std::list<SharedDirInfo> shared_directory_list ;
mSharedDirectories->getSharedDirectoryList(shared_directory_list);
/* recursive update algorithm works that way:
* - the external loop starts on the shared directory list and goes through
* sub-directories
* - at the same time, it updates the local list of shared directories.
* A single sweep is performed over the whole directory structure.
* - the information that is costly to compute (the hash) is stored
* externally into a separate structure.
* - doing so, changing directory names or moving files between directories
* does not cause a re-hash of the content. */
std::list<SharedDirInfo> shared_directory_list;
mSharedDirectories->getSharedDirectoryList(shared_directory_list);
std::set<std::string> sub_dir_list;
std::set<std::string> sub_dir_list ;
/* Support also single files sharing as it make much more sense on some
* platforms like Android */
std::map<std::string, DirectoryStorage::FileTS> singleFilesMap;
// We re-check that each dir actually exists. It might have been removed from the disk.
/* We re-check that each dir actually exists. It might have been removed
* from the disk. Accept also single files not just directories. */
for(auto& realDir: std::as_const(shared_directory_list))
{
const auto& fPath = realDir.filename;
if(RsDirUtil::checkDirectory(fPath))
sub_dir_list.insert(fPath);
else if (RsDirUtil::fileExists(fPath))
{
rstime_t lastWrite= RsDirUtil::lastWriteTime(fPath);
if(time(nullptr) >= lastWrite + MIN_TIME_AFTER_LAST_MODIFICATION)
{
uint64_t fSize = 0;
RsDirUtil::checkFile(fPath,fSize);
for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it)
if(RsDirUtil::checkDirectory( (*real_dir_it).filename ) )
sub_dir_list.insert( (*real_dir_it).filename ) ;
singleFilesMap[fPath].modtime = lastWrite;
singleFilesMap[fPath].size = fSize;
}
else
{
some_files_not_ready = true;
RS_INFO( "file: \"", fPath, "\" is "
"probably being written to. Keep it for later");
}
}
else RS_WARN( "Got non existent file \"", fPath,
"\" in shared directories list. Ignored." );
}
// make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
{
const auto tRoot = mSharedDirectories->root();
std::map<std::string, DirectoryStorage::FileTS> needsUpdate;
mSharedDirectories->updateSubFilesList(
tRoot, singleFilesMap, needsUpdate);
mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list,mHashSalt) ;
for( DirectoryStorage::FileIterator storedSingleFilesIt(
mSharedDirectories, mSharedDirectories->root() );
storedSingleFilesIt; ++storedSingleFilesIt )
{
const auto& it = storedSingleFilesIt;
RsFileHash hash;
if( mHashCache->requestHash(
it.name(), it.size(), it.modtime(), hash,
this, *it ) )
mSharedDirectories->updateHash(*it, hash, it.hash() != hash);
}
}
// now for each of them, go recursively and match both files and dirs
/* make sure that entries in stored_dir_it are the same than paths in
* real_dir_it, and in the same order. */
mSharedDirectories->updateSubDirectoryList(
mSharedDirectories->root(), sub_dir_list, mHashSalt );
std::set<std::string> existing_dirs ;
// now for each of them, go recursively and match both files and dirs
std::set<std::string> existing_dirs;
for( DirectoryStorage::DirIterator stored_dir_it(
mSharedDirectories, mSharedDirectories->root() );
stored_dir_it; ++stored_dir_it )
{
RS_DBG4("recursing into \"", stored_dir_it.name());
for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; stored_dir_it;++stored_dir_it)
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] recursing into " << stored_dir_it.name() << std::endl;
#endif
existing_dirs.insert(RsDirUtil::removeSymLinks(stored_dir_it.name()));
recursUpdateSharedDir(
stored_dir_it.name(), *stored_dir_it,
existing_dirs, 1, some_files_not_ready );
/* here we need to use the list that was stored, instead of the shared
* dir list, because the two are not necessarily in the same order. */
}
recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it,existing_dirs,1,some_files_not_ready) ; // here we need to use the list that was stored, instead of the shared dir list, because the two
// are not necessarily in the same order.
}
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
mIsChecking = false;
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
mIsChecking = false ;
return true ;
return true;
}
void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,std::set<std::string>& existing_directories,uint32_t current_depth,bool& some_files_not_ready)
void LocalDirectoryUpdater::recursUpdateSharedDir(
const std::string& cumulated_path, DirectoryStorage::EntryIndex indx,
std::set<std::string>& existing_directories, uint32_t current_depth,
bool& some_files_not_ready )
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] parsing directory " << cumulated_path << ", index=" << indx << std::endl;
#endif
RS_DBG4("parsing directory \"", cumulated_path, "\" index: ", indx);
// make sure list of subdirs is the same
// make sure list of subfiles is the same
// request all hashes to the hashcache
/* make sure list of subdirs is the same
* make sure list of subfiles is the same
* request all hashes to the hashcache */
librs::util::FolderIterator dirIt(cumulated_path,mFollowSymLinks,false); // disallow symbolic links and files from the future.
// disallow symbolic links and files from the future.
librs::util::FolderIterator dirIt(cumulated_path, mFollowSymLinks, false);
rstime_t dir_local_mod_time ;
if(!mSharedDirectories->getDirectoryLocalModTime(indx,dir_local_mod_time))
{
std::cerr << "(EE) Cannot get local mod time for dir index " << indx << std::endl;
return;
}
rstime_t dir_local_mod_time;
if(!mSharedDirectories->getDirectoryLocalModTime(indx,dir_local_mod_time))
{
RS_ERR("Cannot get local mod time for dir index: ", indx);
print_stacktrace();
return;
}
rstime_t now = time(NULL) ;
rstime_t now = time(nullptr);
/* the > is because we may have changed the virtual name, and therefore the
* TS wont match. We only want to detect when the directory has changed on
* the disk */
if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time)
{
// collect subdirs and subfiles
std::map<std::string, DirectoryStorage::FileTS> subfiles;
std::set<std::string> subdirs;
if(mNeedsFullRecheck || dirIt.dir_modtime() > dir_local_mod_time) // the > is because we may have changed the virtual name, and therefore the TS wont match.
// we only want to detect when the directory has changed on the disk
{
// collect subdirs and subfiles
for( ; dirIt.isValid(); dirIt.next() )
if(filterFile(dirIt.file_name()))
{
const auto fType = dirIt.file_type();
switch(fType)
{
case librs::util::FolderIterator::TYPE_FILE:
if(now >= dirIt.file_modtime() + MIN_TIME_AFTER_LAST_MODIFICATION)
{
subfiles[dirIt.file_name()].modtime = dirIt.file_modtime();
subfiles[dirIt.file_name()].size = dirIt.file_size();
RS_DBG4("adding sub-file \"", dirIt.file_name(), "\"");
}
else
{
some_files_not_ready = true;
RS_INFO( "file: \"", dirIt.file_fullpath(), "\" is "
"probably being written to. Keep it for later");
}
break;
case librs::util::FolderIterator::TYPE_DIR:
{
bool dir_is_accepted = true;
/* 64 is here as a safe limit, to make infinite loops
* impossible.
* TODO: Make it a visible constexpr in the header */
if( (mMaxShareDepth > 0u && current_depth > mMaxShareDepth)
|| (mMaxShareDepth == 0 && current_depth >= 64) )
dir_is_accepted = false;
std::map<std::string,DirectoryStorage::FileTS> subfiles ;
std::set<std::string> subdirs ;
if(dir_is_accepted && mFollowSymLinks && mIgnoreDuplicates)
{
std::string real_path = RsDirUtil::removeSymLinks(
cumulated_path + "/" + dirIt.file_name() );
for(;dirIt.isValid();dirIt.next())
if(filterFile(dirIt.file_name()))
{
switch(dirIt.file_type())
{
case librs::util::FolderIterator::TYPE_FILE:
if( existing_directories.end() !=
existing_directories.find(real_path) )
{
RS_WARN( "Directory: \"", cumulated_path,
"\" has real path: \"", real_path,
"\" which already belongs to another "
"shared directory. Ignoring" );
dir_is_accepted = false;
}
else existing_directories.insert(real_path);
}
if(dirIt.file_modtime() + MIN_TIME_AFTER_LAST_MODIFICATION < now)
{
subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ;
subfiles[dirIt.file_name()].size = dirIt.file_size();
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
#endif
}
else
{
some_files_not_ready = true ;
if(dir_is_accepted) subdirs.insert(dirIt.file_name());
std::cerr << "(WW) file " << dirIt.file_fullpath() << " is probably being written to. Keeping it for later." << std::endl;
}
RS_DBG4("adding sub-dir \"", dirIt.file_name(), "\"");
break;
break;
}
default:
RS_ERR( "Got Dir entry of unknown type:", fType,
"with path \"", cumulated_path, "/",
dirIt.file_name(), "\"" );
print_stacktrace();
break;
}
}
case librs::util::FolderIterator::TYPE_DIR:
{
bool dir_is_accepted = true ;
/* update folder modificatoin time, which is the only way to detect
* e.g. removed or renamed files. */
mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime());
if( (mMaxShareDepth > 0u && current_depth > mMaxShareDepth) || (mMaxShareDepth==0 && current_depth >= 64)) // 64 is here as a safe limit, to make loops impossible.
dir_is_accepted = false ;
// update file and dir lists for current directory.
mSharedDirectories->updateSubDirectoryList(indx,subdirs,mHashSalt);
if(dir_is_accepted && mFollowSymLinks && mIgnoreDuplicates)
{
std::string real_path = RsDirUtil::removeSymLinks(cumulated_path + "/" + dirIt.file_name()) ;
std::map<std::string, DirectoryStorage::FileTS> new_files;
mSharedDirectories->updateSubFilesList(indx, subfiles, new_files);
if(existing_directories.end() != existing_directories.find(real_path))
{
std::cerr << "(WW) Directory " << cumulated_path << " has real path " << real_path << " which already belongs to another shared directory. Ignoring" << std::endl;
dir_is_accepted = false ;
}
else
existing_directories.insert(real_path) ;
}
if(dir_is_accepted)
subdirs.insert(dirIt.file_name());
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
#endif
}
break;
default:
std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
}
}
// update folder modificatoin time, which is the only way to detect e.g. removed or renamed files.
mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime()) ;
// update file and dir lists for current directory.
mSharedDirectories->updateSubDirectoryList(indx,subdirs,mHashSalt) ;
std::map<std::string,DirectoryStorage::FileTS> new_files ;
mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ;
// now go through list of subfiles and request the hash to hashcache
for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit)
{
// ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it.
RsFileHash hash ;
// mSharedDirectories does two things: store H(F), and compute H(H(F)), which is used in FT. The later is always needed.
if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit))
mSharedDirectories->updateHash(*dit,hash,hash != dit.hash());
}
}
#ifdef DEBUG_LOCAL_DIR_UPDATER
else
std::cerr << " directory is unchanged. Keeping existing files and subdirs list." << std::endl;
#endif
// go through the list of sub-dirs and recursively update
for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ; stored_dir_it; ++stored_dir_it)
// now go through list of subfiles and request the hash to hashcache
for( DirectoryStorage::FileIterator dit(mSharedDirectories,indx);
dit; ++dit )
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " recursing into " << stored_dir_it.name() << std::endl;
#endif
recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it,existing_directories,current_depth+1,some_files_not_ready) ;
/* ask about the hash. If not present, ask HashCache.
* If not present, or different, the callback will update it. */
RsFileHash hash;
/* mSharedDirectories does two things: store H(F), and
* compute H(H(F)), which is used in FT.
* The later is always needed. */
if( mHashCache->requestHash(
cumulated_path + "/" + dit.name(),
dit.size(), dit.modtime(), hash, this, *dit ) )
mSharedDirectories->updateHash(*dit, hash, hash != dit.hash());
}
}
// go through the list of sub-dirs and recursively update
for( DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories, indx);
stored_dir_it; ++stored_dir_it )
recursUpdateSharedDir( cumulated_path + "/" + stored_dir_it.name(),
*stored_dir_it, existing_directories,
current_depth+1, some_files_not_ready );
}
bool LocalDirectoryUpdater::filterFile(const std::string& fname) const
@ -347,7 +404,8 @@ void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::stri
bool LocalDirectoryUpdater::hash_confirm(uint32_t client_param)
{
return mSharedDirectories->getEntryType(DirectoryStorage::EntryIndex(client_param)) == DIR_TYPE_FILE ;
return mSharedDirectories->getEntryType(
DirectoryStorage::EntryIndex(client_param) ) == DIR_TYPE_FILE;
}
void LocalDirectoryUpdater::setFileWatchPeriod(int seconds)

25
libretroshare/src/file_sharing/hash_cache.cc
View File

@ -187,9 +187,10 @@ void HashStorage::threadTick()
else
rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", (unsigned long int)mHashCounter+1, (unsigned long int)mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ;
//RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ;
if(rsEvents)
{
/* Emit deprecated event only for retrocompatibility
* TODO: create a proper event with structured data instead of a
* formatted string */
auto ev = std::make_shared<RsSharedDirectoriesEvent>();
ev->mEventCode = RsSharedDirectoriesEventCode::HASHING_FILE;
ev->mMessage = tmpout;
@ -198,7 +199,7 @@ void HashStorage::threadTick()
double seconds_origin = rstime::RsScopeTimer::currentTime() ;
if(RsDirUtil::getFileHash(job.full_path, hash,size, this))
if(RsDirUtil::getFileHash(job.full_path, hash, size, this))
{
// store the result
@ -218,8 +219,7 @@ void HashStorage::threadTick()
mChanged = true ;
mTotalHashedSize += size ;
}
else
std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl;
else RS_ERR("Failure hashing file: ", job.full_path);
mHashingTime += rstime::RsScopeTimer::currentTime() - seconds_origin ;
mHashedBytes += size ;
@ -233,11 +233,18 @@ void HashStorage::threadTick()
++mHashCounter ;
}
}
// call the client
}
if(!hash.isNull())
job.client->hash_callback(job.client_param, job.full_path, hash, size);
// call the client
if(!hash.isNull())
job.client->hash_callback(job.client_param, job.full_path, hash, size);
/* Notify we completed hashing a file */
auto ev = std::make_shared<RsFileHashingCompletedEvent>();
ev->mFilePath = job.full_path;
ev->mHashingSpeed = mCurrentHashingSpeed;
ev->mFileHash = hash;
rsEvents->postEvent(ev);
}
bool HashStorage::requestHash(const std::string& full_path,uint64_t size,rstime_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param)

344
libretroshare/src/file_sharing/p3filelists.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2018 Mr.Alice <mralice@users.sourceforge.net> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -30,7 +32,7 @@
#include "retroshare/rsids.h"
#include "retroshare/rspeers.h"
#include "retroshare/rsinit.h"
#include "util/cxx17retrocompat.h"
#include "rsserver/p3face.h"
#define P3FILELISTS_DEBUG() std::cerr << time(NULL) << " : FILE_LISTS : " << __FUNCTION__ << " : "
@ -1044,31 +1046,36 @@ void p3FileDatabase::getExtraFilesDirDetails(void *ref,DirectoryStorage::EntryIn
}
// This function converts a pointer into directory details, to be used by the AbstractItemModel for browsing the files.
int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags flags) const
int p3FileDatabase::RequestDirDetails(
void* ref, DirDetails& d, FileSearchFlags flags ) const
{
RS_STACK_MUTEX(mFLSMtx) ;
RS_STACK_MUTEX(mFLSMtx);
d.children.clear();
d.children.clear();
// Case where the pointer is NULL, which means we're at the top of the list of shared directories for all friends (including us)
// or at the top of our own list of shared directories, depending on the flags.
/* Case where the pointer is NULL, which means we're at the top of the list
* of shared directories for all friends (including us) or at the top of our
* own list of shared directories, depending on the flags.
*
* Friend index is used as follows:
* 0 : own id
* 1...n : other friends
*
* entry_index: starts at 0.
*
* The point is: we cannot use (0,0) because it encodes to NULL. No existing
* combination should encode to NULL.
* So we need to properly convert the friend index into 0 or into a friend
* tab index in mRemoteDirectories.
*
* We should also check the consistency between flags and the content of ref.
*/
// Friend index is used as follows:
// 0 : own id
// 1...n : other friends
//
// entry_index: starts at 0.
//
// The point is: we cannot use (0,0) because it encodes to NULL. No existing combination should encode to NULL.
// So we need to properly convert the friend index into 0 or into a friend tab index in mRemoteDirectories.
//
// We should also check the consistency between flags and the content of ref.
if (ref == NULL)
{
d.ref = NULL ;
d.type = DIR_TYPE_ROOT;
d.parent = NULL;
if (ref == nullptr)
{
d.ref = nullptr;
d.type = DIR_TYPE_ROOT;
d.parent = nullptr;
d.prow = -1;
d.name = "root";
d.hash.clear() ;
@ -1078,12 +1085,13 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
d.max_mtime = 0 ;
if(flags & RS_FILE_HINTS_LOCAL)
{
void *p;
{
void *p = nullptr;
{
convertEntryIndexToPointer<sizeof(void*)>(0,0,p); // root of own directories
DirStub stub;
{
// root of own directories
convertEntryIndexToPointer<sizeof(void*)>(0, 0, p);
DirStub stub;
stub.type = DIR_TYPE_PERSON;
stub.name = mServCtrl->getOwnId().toStdString();
stub.ref = p;
@ -1092,9 +1100,11 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
if(mExtraFiles->size() > 0)
{
convertEntryIndexToPointer<sizeof(void*)>(0,1,p); // local shared files from extra list
DirStub stub;
stub.type = DIR_TYPE_PERSON; // not totally exact, but used as a trick.
// local shared files from extra list
convertEntryIndexToPointer<sizeof(void*)>(0, 1, p);
DirStub stub;
// not totally exact, but used as a trick.
stub.type = DIR_TYPE_PERSON;
stub.name = "[Extra List]";
stub.ref = p;
@ -1127,18 +1137,19 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
}
uint32_t fi;
DirectoryStorage::EntryIndex e ;
DirectoryStorage::EntryIndex e;
convertPointerToEntryIndex<sizeof(void*)>(ref,e,fi);
// check consistency
if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 1 && (flags & RS_FILE_HINTS_LOCAL)))
{
P3FILELISTS_ERROR() << "(EE) remote request on local index or local request on remote index. This should not happen." << std::endl;
return false ;
}
// check consistency
if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) ||
(fi > 1 && (flags & RS_FILE_HINTS_LOCAL)))
{
RS_ERR("Remote request on local index or local request on remote index");
return false;
}
if((flags & RS_FILE_HINTS_LOCAL) && fi == 1) // extra list
if((flags & RS_FILE_HINTS_LOCAL) && fi == 1) // extra list
{
getExtraFilesDirDetails(ref,e,d);
@ -1150,25 +1161,28 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
}
DirectoryStorage *storage = (flags & RS_FILE_HINTS_LOCAL)? ((DirectoryStorage*)mLocalSharedDirs) : ((DirectoryStorage*)mRemoteDirectories[fi-1]);
DirectoryStorage* storage =
(flags & RS_FILE_HINTS_LOCAL) ?
((DirectoryStorage*) mLocalSharedDirs) :
((DirectoryStorage*) mRemoteDirectories[fi-1]);
// Case where the index is the top of a single person. Can be us, or a friend.
/* Case where the index is the top of a single person.
* Can be us, or a friend. */
if(!storage || !storage->extractData(e,d))
{
RS_WARN( "request on index; ", e, ", for directory ID:",
( (!storage)? ("[NULL]") : (storage->peerId().toStdString()) ),
" failed" );
return false;
}
if(storage==NULL || !storage->extractData(e,d))
{
#ifdef DEBUG_FILE_HIERARCHY
P3FILELISTS_DEBUG() << "(WW) request on index " << e << ", for directory ID=" << ((storage==NULL)?("[NULL]"):(storage->peerId().toStdString())) << " failed. This should not happen." << std::endl;
#endif
return false ;
}
/* update indexes. This is a bit hacky, but does the job. The cast to
* intptr_t is the proper way to convert a pointer into an int. */
convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.ref,fi,d.ref);
// update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert
// a pointer into an int.
convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.ref,fi,d.ref) ;
for(uint32_t i=0;i<d.children.size();++i)
convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.children[i].ref,fi,d.children[i].ref);
for(uint32_t i=0; i<d.children.size(); ++i)
convertEntryIndexToPointer<sizeof(void*)>(
(intptr_t) d.children[i].ref, fi, d.children[i].ref );
if(e == 0) // root
{
@ -1178,9 +1192,9 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
else
{
if(d.parent == 0) // child of root node
d.prow = (flags & RS_FILE_HINTS_LOCAL)?0:(fi-1);
d.prow = (flags & RS_FILE_HINTS_LOCAL) ? 0 : (fi-1);
else
d.prow = storage->parentRow(e) ;
d.prow = storage->parentRow(e);
convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.parent,fi,d.parent) ;
}
@ -1307,7 +1321,9 @@ uint32_t p3FileDatabase::watchPeriod()
return mLocalDirWatcher->fileWatchPeriod();
}
int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& client_peer_id)
int p3FileDatabase::SearchKeywords(
const std::list<std::string>& keywords, std::list<DirDetails>& results,
FileSearchFlags flags, const RsPeerId& client_peer_id )
{
if(flags & RS_FILE_HINTS_LOCAL)
{
@ -1319,15 +1335,15 @@ int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::
mLocalSharedDirs->searchTerms(keywords,firesults) ;
for(std::list<EntryIndex>::iterator it(firesults.begin());it!=firesults.end();++it)
{
void *p=NULL;
convertEntryIndexToPointer<sizeof(void*)>(*it,0,p);
pointers.push_back(p) ;
}
for(auto& it: std::as_const(firesults))
{
void *p = nullptr;
convertEntryIndexToPointer<sizeof(void*)>(it, 0, p);
pointers.push_back(p);
}
}
filterResults(pointers,results,flags,client_peer_id) ;
filterResults(pointers, results, flags, client_peer_id);
}
if(flags & RS_FILE_HINTS_REMOTE)
@ -1476,7 +1492,9 @@ bool p3FileDatabase::search(
return false;
}
int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const
int p3FileDatabase::filterResults(
const std::list<void*>& firesults, std::list<DirDetails>& results,
FileSearchFlags flags, const RsPeerId& peer_id ) const
{
results.clear();
@ -1484,29 +1502,33 @@ int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<Di
/* translate/filter results */
for(std::list<void*>::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit)
{
DirDetails cdetails ;
for(void* rit: std::as_const(firesults))
{
DirDetails cdetails;
if(!RequestDirDetails (*rit,cdetails,RS_FILE_HINTS_LOCAL))
{
P3FILELISTS_ERROR() << "(EE) Cannot get dir details for entry " << *rit << std::endl;
continue ;
}
if(!RequestDirDetails(rit, cdetails, RS_FILE_HINTS_LOCAL))
{
RS_ERR("Cannot retrieve dir details for entry: ", rit);
print_stacktrace();
continue ;
}
RS_DBG( "Filtering candidate: ", rit,
", name: ", cdetails.name, ", path: ", cdetails.path,
", flags: ", cdetails.flags, ", peer: ", peer_id );
if(!peer_id.isNull())
{
FileSearchFlags permission_flags =
rsPeers->computePeerPermissionFlags(
peer_id, cdetails.flags, cdetails.parent_groups );
if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags ))
{
cdetails.id.clear();
results.push_back(cdetails);
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Filtering candidate " << *rit << ", flags=" << cdetails.flags << ", peer=" << peer_id ;
#endif
if(!peer_id.isNull())
{
FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(peer_id,cdetails.flags,cdetails.parent_groups) ;
if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags ))
{
cdetails.id.clear() ;
results.push_back(cdetails);
#ifdef DEBUG_P3FILELISTS
std::cerr << ": kept" << std::endl ;
std::cerr << ": kept" << std::endl ;
#endif
}
#ifdef DEBUG_P3FILELISTS
@ -1518,7 +1540,19 @@ int p3FileDatabase::filterResults(const std::list<void*>& firesults,std::list<Di
results.push_back(cdetails);
}
return !results.empty() ;
for(auto& res: results)
{
/* Most file will just have file name stored, but single file shared
* without a shared dir will contain full path instead of just the
* name, so purify it to perform the search */
if(res.name.find("/") != std::string::npos )
{
std::string _tDirPath;
RsDirUtil::splitDirFromFile(res.name, _tDirPath, res.name);
}
}
return !results.empty();
}
bool p3FileDatabase::convertSharedFilePath(const std::string& path,std::string& fullpath)
@ -1592,78 +1626,90 @@ void p3FileDatabase::tickSend()
void p3FileDatabase::handleDirSyncRequest(RsFileListsSyncRequestItem *item)
{
RsFileListsSyncResponseItem *ritem = new RsFileListsSyncResponseItem;
RsFileListsSyncResponseItem* ritem = new RsFileListsSyncResponseItem;
// look at item TS. If local is newer, send the full directory content.
{
RS_STACK_MUTEX(mFLSMtx) ;
// look at item TS. If local is newer, send the full directory content.
{
RS_STACK_MUTEX(mFLSMtx);
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << "Received directory sync request from peer " << item->PeerId() << ". hash=" << item->entry_hash << ", flags=" << (void*)(intptr_t)item->flags << ", request id: " << std::hex << item->request_id << std::dec << ", last known TS: " << item->last_known_recurs_modf_TS << std::endl;
#endif
RS_DBG( "Received directory sync request from peer ", item->PeerId(),
". hash=", item->entry_hash, ", flags=", item->flags,
", request id: ", item->request_id, ", last known TS: ",
item->last_known_recurs_modf_TS );
EntryIndex entry_index = DirectoryStorage::NO_INDEX;
EntryIndex entry_index = DirectoryStorage::NO_INDEX;
if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index))
{
RS_DBG("Cannot find entry index for hash ", item->entry_hash,
" cannot respond to sync request." );
return;
}
if(!mLocalSharedDirs->getIndexFromDirHash(item->entry_hash,entry_index))
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " (EE) Cannot find entry index for hash " << item->entry_hash << ": cannot respond to sync request." << std::endl;
#endif
return;
}
uint32_t entry_type = mLocalSharedDirs->getEntryType(entry_index);
ritem->PeerId(item->PeerId());
ritem->request_id = item->request_id;
ritem->entry_hash = item->entry_hash;
uint32_t entry_type = mLocalSharedDirs->getEntryType(entry_index) ;
ritem->PeerId(item->PeerId()) ;
ritem->request_id = item->request_id;
ritem->entry_hash = item->entry_hash ;
std::list<RsNodeGroupId> node_groups;
FileStorageFlags node_flags;
std::list<RsNodeGroupId> node_groups;
FileStorageFlags node_flags;
if(entry_type != DIR_TYPE_DIR)
{
RS_DBG( "Directory does not exist anymore, or is not a directory, "
"or permission denied. Answering with proper flags." );
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED;
}
else if( entry_index != 0 &&
(!mLocalSharedDirs->getFileSharingPermissions(
entry_index, node_flags,node_groups ) ||
!(rsPeers->computePeerPermissionFlags(
item->PeerId(), node_flags, node_groups ) &
RS_FILE_HINTS_BROWSABLE)) )
{
RS_ERR("cannot get file permissions for entry index: ", entry_index,
", or permission denied." );
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED;
}
else
{
rstime_t local_recurs_max_time;
mLocalSharedDirs->getDirectoryRecursModTime(
entry_index, local_recurs_max_time );
if(entry_type != DIR_TYPE_DIR)
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Directory does not exist anymore, or is not a directory, or permission denied. Answering with proper flags." << std::endl;
#endif
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
}
else if(entry_index != 0 && (!mLocalSharedDirs->getFileSharingPermissions(entry_index,node_flags,node_groups) || !(rsPeers->computePeerPermissionFlags(item->PeerId(),node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
{
P3FILELISTS_ERROR() << "(EE) cannot get file permissions for entry index " << (void*)(intptr_t)entry_index << ", or permission denied." << std::endl;
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_WAS_REMOVED ;
}
else
{
rstime_t local_recurs_max_time ;
mLocalSharedDirs->getDirectoryRecursModTime(entry_index,local_recurs_max_time) ;
/* normally, should be "<", but since we provided the TS it should
* be equal, so != is more robust. */
if(item->last_known_recurs_modf_TS != local_recurs_max_time)
{
RS_DBG( "Directory is more recent than what the friend knows. "
"Sending full dir content as response.");
if(item->last_known_recurs_modf_TS != local_recurs_max_time) // normally, should be "<", but since we provided the TS it should be equal, so != is more robust.
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Directory is more recent than what the friend knows. Sending full dir content as response." << std::endl;
#endif
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
RsFileListsItem::FLAGS_SYNC_DIR_CONTENT;
ritem->last_known_recurs_modf_TS = local_recurs_max_time;
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_SYNC_DIR_CONTENT;
ritem->last_known_recurs_modf_TS = local_recurs_max_time;
/* We supply the peer id, in order to possibly remove some
* subdirs, if entries are not allowed to be seen by this peer.
*/
mLocalSharedDirs->serialiseDirEntry(
entry_index, ritem->directory_content_data,
item->PeerId() );
}
else
{
RS_DBG3( "Directory is up to date w.r.t. what the friend knows."
" Sending ACK." );
// We supply the peer id, in order to possibly remove some subdirs, if entries are not allowed to be seen by this peer.
mLocalSharedDirs->serialiseDirEntry(entry_index,ritem->directory_content_data,item->PeerId()) ;
}
else
{
#ifdef DEBUG_P3FILELISTS
P3FILELISTS_DEBUG() << " Directory is up to date w.r.t. what the friend knows. Sending ACK." << std::endl;
#endif
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE |
RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE;
ritem->last_known_recurs_modf_TS = local_recurs_max_time;
}
}
}
ritem->flags = RsFileListsItem::FLAGS_SYNC_RESPONSE | RsFileListsItem::FLAGS_ENTRY_UP_TO_DATE ;
ritem->last_known_recurs_modf_TS = local_recurs_max_time ;
}
}
}
// sends the response.
splitAndSendItem(ritem) ;
// sends the response.
splitAndSendItem(ritem);
}
void p3FileDatabase::splitAndSendItem(RsFileListsSyncResponseItem *ritem)

12
libretroshare/src/ft/ftserver.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2008 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -876,9 +878,11 @@ int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetai
{
return mFileDatabase->SearchKeywords(keywords, results,flags,RsPeerId());
}
int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id)
int ftServer::SearchKeywords(
std::list<std::string> keywords, std::list<DirDetails> &results,
FileSearchFlags flags, const RsPeerId& peer_id )
{
return mFileDatabase->SearchKeywords(keywords, results,flags,peer_id);
return mFileDatabase->SearchKeywords(keywords, results, flags, peer_id);
}
int ftServer::SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags)
@ -2033,7 +2037,7 @@ bool ftServer::receiveSearchRequest(
std::vector<DeepFilesSearchResult> dRes;
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0)
if(!dfi.search(searchReq.queryString, dRes, maxAllowsHits))
{
RsFileSearchResultItem resIt;

63
libretroshare/src/gxs/rsgenexchange.cc
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -193,47 +194,6 @@ RsGenExchange::RsGenExchange(
VALIDATE_MAX_WAITING_TIME(60)
{
mDataAccess = new RsGxsDataAccess(gds);
// Perform an early checking/cleaning of the db. Will eliminate groups and messages that do not match their hash
#ifdef RS_DEEP_CHANNEL_INDEX
// This code is only called because it of deep indexing in channels. But loading
// the entire set of messages in order to provide indexing is pretty bad (very costly and slowly
// eats memory, as many tests have shown. Not because of leaks, but because new threads are
// apparently attributed large stacks and pages of memory are not regained by the system maybe because it thinks
// that RS will use them again.
//
// * the deep check should be implemented differently, in an incremental way. For instance in notifyChanges() of each
// service (e.g. channels here) should update the index when a new message is received. The question to how old messages
// are processed is open. I believe that they shouldn't. New users will progressively process them.
//
// * integrity check (re-hashing of message data) is not needed. Message signature already ensures that all messages received are
// unalterated. The only problem (possibly very rare) is that a message is locally corrupted and not deleted (because of no check).
// It will therefore never be replaced by the correct one from friends. The cost of re-hashing the whole db data regularly
// doesn't counterbalance such a low risk.
//
if(mServType == RS_SERVICE_GXS_TYPE_CHANNELS)
{
std::vector<RsGxsGroupId> grpsToDel;
GxsMsgReq msgsToDel;
RsGxsSinglePassIntegrityCheck::check(mServType,mGixs,mDataStore,
this, *mSerialiser,
grpsToDel,msgsToDel);
for(auto& grpId: grpsToDel)
{
uint32_t token2=0;
deleteGroup(token2,grpId);
}
if(!msgsToDel.empty())
{
uint32_t token1=0;
deleteMsgs(token1,msgsToDel);
}
}
#endif
}
void RsGenExchange::setNetworkExchangeService(RsNetworkExchangeService *ns)
@ -362,12 +322,12 @@ void RsGenExchange::tick()
{
mIntegrityCheck = new RsGxsIntegrityCheck( mDataStore, this,
*mSerialiser, mGixs);
mIntegrityCheck->start("gxs integrity");
mChecking = true;
std::string thisName = typeid(*this).name();
mChecking = mIntegrityCheck->start("gxs IC4 "+thisName);
}
}
if(mIntegrityCheck->isDone())
if(mIntegrityCheck->isDone() || !mChecking)
{
std::vector<RsGxsGroupId> grpIds;
GxsMsgReq msgIds;
@ -2717,10 +2677,15 @@ void RsGenExchange::processMessageDelete()
msgDeleted.push_back(note.msgIds);
}
for(const auto& msgreq:msgDeleted)
for(const auto& msgit:msgreq)
for(const auto& msg:msgit.second)
mNotifications.push_back(new RsGxsMsgChange(RsGxsNotify::TYPE_MESSAGE_DELETED,msgit.first,msg, false));
/* Three nested for looks like a performance bomb, but as Cyril says here
* https://github.com/RetroShare/RetroShare/pull/2218#pullrequestreview-565194022
* this should actually not explode at all because it is just one message at
* time that get notified */
for(const auto& msd : mMsgDeletePublish)
for(auto& msgMap : msd.mMsgs)
for(auto& msgId : msgMap.second)
mNotifications.push_back(
new RsGxsMsgDeletedChange(msgMap.first, msgId) );
mMsgDeletePublish.clear();
}

140
libretroshare/src/gxs/rsgxsnetservice.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Christopher Evi-Parker *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -5204,13 +5206,14 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st
return mGxsNetTunnel->turtleSearchRequest(match_string,this) ;
}
#ifndef RS_DEEP_CHANNEL_INDEX
static bool termSearch(const std::string& src, const std::string& substring)
{
/* always ignore case */
return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() );
/* always ignore case */
return src.end() != std::search(
src.begin(), src.end(), substring.begin(), substring.end(),
RsRegularExpression::CompareCharIC() );
}
#endif // ndef RS_DEEP_CHANNEL_INDEX
bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSearchResults>& group_infos)
{
@ -5246,7 +5249,8 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
return true ;
}
void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
void RsGxsNetService::receiveTurtleSearchResults(
TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos )
{
std::set<RsGxsGroupId> groupsToNotifyResults;
@ -5276,26 +5280,20 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
for (const RsGxsGroupSummary& gps : group_infos)
{
#ifndef RS_DEEP_CHANNEL_INDEX
#ifdef TO_REMOVE
/* Because of deep search is enabled search results may bring more
* info then we already have also about post that are indexed by
* xapian, so we don't apply this filter anymore. */
/* Only keep groups that are not locally known, and groups that are
* not already in the mDistantSearchResults structure.
* mDataStore may in some situations allocate an empty group meta data, so it's important
* to test that the group meta is both non null and actually corresponds to the group id we seek. */
* mDataStore may in some situations allocate an empty group meta
* data, so it's important to test that the group meta is both non
* null and actually corresponds to the group id we seek. */
auto& meta(grpMeta[gps.mGroupId]);
if(meta != nullptr && meta->mGroupId == gps.mGroupId) continue;
#endif // def TO_REMOVE
auto& meta(grpMeta[gps.mGroupId]);
if(meta != nullptr && meta->mGroupId == gps.mGroupId)
continue;
#ifdef NXS_NET_DEBUG_9
std::cerr << " group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl;
#endif
#else // ndef RS_DEEP_CHANNEL_INDEX
/* When deep search is enabled search results may bring more info
* then we already have also about post that are indexed by xapian,
* so we don't apply this filter in this case. */
#endif
const RsGxsGroupId& grpId(gps.mGroupId);
groupsToNotifyResults.insert(grpId);
@ -5332,18 +5330,19 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std
mObserver->receiveDistantSearchResults(req, grpId);
}
void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)
void RsGxsNetService::receiveTurtleSearchResults(
TurtleRequestId req,
const uint8_t* encrypted_group_data, uint32_t encrypted_group_data_len )
{
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl;
#endif
auto it = mSearchRequests.find(req);
if(mSearchRequests.end() == it)
{
std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ;
return;
}
auto it = mSearchRequests.find(req);
if(mSearchRequests.end() == it)
{
RS_WARN("Received search results for unknown request: ", req);
return;
}
RsGxsGroupId grpId = it->second;
uint8_t encryption_master_key[32];
@ -5417,56 +5416,42 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsig
mObserver->receiveDistantSearchResults(req, grpId);
}
std::error_condition RsGxsNetService::distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId )
{
if(!mGxsNetTunnel)
{
free(searchData);
return std::errc::function_not_supported;
}
return mGxsNetTunnel->turtleSearchRequest(
searchData, dataSize, serviceType, requestId );
}
std::error_condition RsGxsNetService::handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
RS_DBG("");
return mObserver->handleDistantSearchRequest(
requestData, requestSize, resultData, resultSize );
}
std::error_condition RsGxsNetService::receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
return mObserver->receiveDistantSearchResult(
requestId, resultData, resultSize );
}
bool RsGxsNetService::search( const std::string& substring,
std::list<RsGxsGroupSummary>& group_infos )
{
group_infos.clear();
#ifdef RS_DEEP_CHANNEL_INDEX
#warning TODO: filter deep index search result to non circle-restricted groups.
// /!\
// /!\ These results should be filtered to only return results coming from a non restricted group!
// /!\
std::vector<DeepChannelsSearchResult> results;
DeepChannelsIndex::search(substring, results);
for(auto dsr : results)
{
RsUrl rUrl(dsr.mUrl);
const auto& uQ(rUrl.query());
auto rit = uQ.find("id");
if(rit != rUrl.query().end())
{
RsGroupNetworkStats stats;
RsGxsGroupId grpId(rit->second);
if( !grpId.isNull() && getGroupNetworkStats(grpId, stats) )
{
RsGxsGroupSummary s;
s.mGroupId = grpId;
if((rit = uQ.find("name")) != uQ.end())
s.mGroupName = rit->second;
if((rit = uQ.find("signFlags")) != uQ.end())
s.mSignFlags = static_cast<uint32_t>(std::stoul(rit->second));
if((rit = uQ.find("publishTs")) != uQ.end())
s.mPublishTs = static_cast<rstime_t>(std::stoll(rit->second));
if((rit = uQ.find("authorId")) != uQ.end())
s.mAuthorId = RsGxsId(rit->second);
s.mSearchContext = dsr.mSnippet;
s.mNumberOfMessages = stats.mMaxVisibleCount;
s.mLastMessageTs = stats.mLastGroupModificationTS;
s.mPopularity = stats.mSuppliers;
group_infos.push_back(s);
}
}
}
#else // RS_DEEP_CHANNEL_INDEX
RsGxsGrpMetaTemporaryMap grpMetaMap;
{
RS_STACK_MUTEX(mNxsMutex) ;
@ -5492,12 +5477,11 @@ bool RsGxsNetService::search( const std::string& substring,
group_infos.push_back(s);
}
#endif // RS_DEEP_CHANNEL_INDEX
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl;
#endif
return !group_infos.empty();
return !group_infos.empty();
}
bool RsGxsNetService::search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)

62
libretroshare/src/gxs/rsgxsnetservice.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Christopher Evi-Parker *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSGXSNETSERVICE_H
#define RSGXSNETSERVICE_H
#pragma once
#include <list>
#include <queue>
@ -130,18 +131,53 @@ public:
virtual bool msgAutoSync() const override { return mAllowMsgSync; }
virtual bool grpAutoSync() const override { return mGrpAutoSync; }
/*!
* \brief Search methods.
* These four methods are used to request distant search and receive the results.
* \param group_id
*/
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)override ;
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)override ;
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) override ;
/// @see RsNetworkExchangeService
std::error_condition distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId ) override;
/// @see RsNetworkExchangeService
std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/// @see RsNetworkExchangeService
std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/** Request group data via turtle search
* @param group_id */
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override;
/**
* @brief Search for matching groups names over turtle search.
* @deprecated this method is kept mostly for retrocompatibility with older
* peers, newly implemented search functions should instead be based on the
* service generic search.
* @see RsNetworkExchangeService
*/
RS_DEPRECATED_FOR(distantSearchRequest)
TurtleRequestId turtleSearchRequest(const std::string& match_string) override;
/** @see RsNetworkExchangeService
* @deprecated kept for retrocompatibility with older peers, new code should
* instead be based on the service generic search */
RS_DEPRECATED_FOR(receiveDistantSearchResult)
void receiveTurtleSearchResults(
TurtleRequestId req,
const uint8_t* encrypted_group_data,
uint32_t encrypted_group_data_len ) override;
/**
* @deprecated kept for retrocompatibility with older peers, new code should
* instead be based on the service generic search */
RS_DEPRECATED_FOR(handleRemoteSearchRequest)
virtual bool search( const std::string& substring,
std::list<RsGxsGroupSummary>& group_infos) override;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)override ;
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)override ;
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)override ;
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSearchResults> &group_infos)override ;
virtual bool clearDistantSearchResults(const TurtleRequestId& id)override ;
@ -629,5 +665,3 @@ private:
bool mUseMetaCache;
};
#endif // RSGXSNETSERVICE_H

607
libretroshare/src/gxs/rsgxsnettunnel.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2018 Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -27,6 +29,8 @@
#include "gxs/rsnxs.h"
#include "rsgxsnettunnel.h"
/*extern*/ RsGxsDistSync* rsGxsDistSync = nullptr;
//#define DEBUG_RSGXSNETTUNNEL 1
#define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
@ -36,42 +40,93 @@
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA = 1;
static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH = 100;
RsGxsDistSync *rsGxsDistSync = NULL;
RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel")
{
mRandomBias.clear();
mLastKeepAlive = time(NULL) + (RSRandom::random_u32()%20); // adds some variance in order to avoid doing all this tasks at once across services
mLastAutoWash = time(NULL) + (RSRandom::random_u32()%20);
mLastDump = time(NULL) + (RSRandom::random_u32()%20);
/* adds some variance in order to avoid doing all this tasks at once across
* services */
auto now = time(nullptr);
mLastKeepAlive = now + (RsRandom::random_u32()%20);
mLastAutoWash = now + (RsRandom::random_u32()%20);
mLastDump = now + (RsRandom::random_u32()%20);
}
//===========================================================================================================================================//
// Transport Items //
//===========================================================================================================================================//
//============================================================================//
// Transport Items //
//============================================================================//
const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ;
enum class RsGxsNetTunnelItemSubtypes : uint8_t
{
VIRTUAL_PEER = 0x01,
KEEP_ALIVE = 0x02,
RANDOM_BIAS = 0x03,
/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REQUEST
SEARCH_SUBSTRING = 0x04,
SEARCH_GROUP_REQUEST = 0x05,
// SEARCH_GROUP_SUMMARY = 0x06, removed
SEARCH_GROUP_DATA = 0x07,
/// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REPLY
SEARCH_GROUP_SUMMARY = 0x08,
/** Generic search request generated and handled by specific service
* (channels, forums...) */
SERVICE_SEARCH_REQUEST = 0x09,
/** Generic search reply generated and handled by specific service
* (channels, forums...) */
SERVICE_SEARCH_REPLY = 0x0a
};
RS_DEPRECATED_FOR(RsServiceType::GXS_DISTANT)
constexpr uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL =
static_cast<uint16_t>(RsServiceType::GXS_DISTANT);
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING = 0x04 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ;
// const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x06; // DEPRECATED
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA = 0x07 ;
RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes)
const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x08;
// Do not add new subitems types as const, use RsGxsNetTunnelItemSubtypes instead
class RsGxsNetTunnelItem: public RsItem
struct RsGxsNetTunnelItem: RsItem
{
public:
explicit RsGxsNetTunnelItem(uint8_t item_subtype) : RsItem(RS_PKT_VERSION_SERVICE,RS_SERVICE_TYPE_GXS_NET_TUNNEL,item_subtype)
explicit RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype):
RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
static_cast<uint8_t>(subtype) )
{
/* no priority. All items are encapsulated into generic Turtle items
* anyway. */
}
virtual ~RsGxsNetTunnelItem() = default;
virtual void clear() {}
RS_DEPRECATED_FOR("RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype)")
explicit RsGxsNetTunnelItem(uint8_t item_subtype):
RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL,
item_subtype )
{
// no priority. All items are encapsulated into generic Turtle items anyway.
}
virtual ~RsGxsNetTunnelItem() {}
virtual void clear() {}
};
class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem
@ -113,7 +168,86 @@ public:
Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization.
};
class RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
struct RsGxsServiceTurtleSearchReqItem: RsGxsNetTunnelItem
{
RsGxsServiceTurtleSearchReqItem():
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
mServiceType(RsServiceType::NONE), mSearchData(nullptr),
mSearchDataSize(0) {}
explicit RsGxsServiceTurtleSearchReqItem(RsServiceType service):
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST),
mServiceType(service), mSearchData(nullptr), mSearchDataSize(0) {}
/// Type of the service which originated the search request
RsServiceType mServiceType;
uint8_t* mSearchData; /// Service search request data
uint32_t mSearchDataSize; /// Search data size
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mServiceType);
RsTypeSerializer::RawMemoryWrapper prox(mSearchData, mSearchDataSize);
RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
}
/// @see RsItem
void clear() override
{
free(mSearchData);
mSearchData = nullptr;
mSearchDataSize = 0;
}
~RsGxsServiceTurtleSearchReqItem() override { clear(); }
};
struct RsGxsServiceTurtleSearchReplyItem: RsGxsNetTunnelItem
{
RsGxsServiceTurtleSearchReplyItem():
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
mServiceType(RsServiceType::NONE), mReplyData(nullptr),
mReplyDataSize(0) {}
explicit RsGxsServiceTurtleSearchReplyItem(RsServiceType service):
RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY),
mServiceType(service), mReplyData(nullptr), mReplyDataSize(0) {}
/// Type of the service which originated the search request
RsServiceType mServiceType;
uint8_t* mReplyData; /// Service search reply data
uint32_t mReplyDataSize; /// Search reply data size
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mServiceType);
RsTypeSerializer::RawMemoryWrapper prox(mReplyData, mReplyDataSize);
RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData");
}
/// @see RsItem
void clear() override
{
free(mReplyData);
mReplyData = nullptr;
mReplyDataSize = 0;
}
~RsGxsServiceTurtleSearchReplyItem() override { clear(); }
};
class RS_DEPRECATED_FOR(RsGxsServiceTurtleSearchItem)
RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem
{
public:
explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {}
@ -164,6 +298,7 @@ public:
RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ;
}
};
class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem
{
public:
@ -193,28 +328,41 @@ public:
class RsGxsNetTunnelSerializer: public RsServiceSerializer
{
public:
RsGxsNetTunnelSerializer() :RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
RsGxsNetTunnelSerializer():
RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {}
virtual RsItem *create_item(uint16_t service,uint8_t item_subtype) const
{
if(service != RS_SERVICE_TYPE_GXS_NET_TUNNEL)
{
GXS_NET_TUNNEL_ERROR() << "received item with wrong service ID " << std::hex << service << std::dec << std::endl;
return NULL ;
RS_ERR( "received item with wrong service ID ", service);
print_stacktrace();
return nullptr;
}
switch(item_subtype)
switch(static_cast<RsGxsNetTunnelItemSubtypes>(item_subtype))
{
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER : return new RsGxsNetTunnelVirtualPeerItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE : return new RsGxsNetTunnelKeepAliveItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS : return new RsGxsNetTunnelRandomBiasItem ;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING : return new RsGxsNetTunnelTurtleSearchSubstringItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST : return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY : return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA : return new RsGxsNetTunnelTurtleSearchGroupDataItem;
case RsGxsNetTunnelItemSubtypes::VIRTUAL_PEER:
return new RsGxsNetTunnelVirtualPeerItem;
case RsGxsNetTunnelItemSubtypes::KEEP_ALIVE:
return new RsGxsNetTunnelKeepAliveItem;
case RsGxsNetTunnelItemSubtypes::RANDOM_BIAS:
return new RsGxsNetTunnelRandomBiasItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
return new RsGxsNetTunnelTurtleSearchSubstringItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
return new RsGxsNetTunnelTurtleSearchGroupRequestItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
return new RsGxsNetTunnelTurtleSearchGroupSummaryItem;
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
return new RsGxsNetTunnelTurtleSearchGroupDataItem;
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
return new RsGxsServiceTurtleSearchReqItem;
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
return new RsGxsServiceTurtleSearchReplyItem;
default:
GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << (int)item_subtype << std::dec << " is not handled!" << std::endl;
return NULL ;
RS_ERR("Unkonown item type: ", static_cast<int>(item_subtype));
return nullptr;
}
}
};
@ -993,7 +1141,9 @@ TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& gr
return mTurtle->turtleSearch(mem,size,this) ;
}
TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service)
TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(
const std::string& match_string,
RsNetworkExchangeService* client_service )
{
GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string \"" << match_string << "\"" << std::endl;
@ -1002,7 +1152,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
search_item.service = client_service->serviceType() ;
uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ;
unsigned char *mem = (unsigned char*)rs_malloc(size) ;
uint8_t* mem = rs_malloc<uint8_t>(size);
if(mem == NULL)
return 0 ;
@ -1013,151 +1163,304 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma
return mTurtle->turtleSearch(mem,size,this) ;
}
bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits)
std::error_condition RsGxsNetTunnelService::turtleSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId )
{
GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl;
if(!searchData || !dataSize || serviceType == RsServiceType::NONE)
return std::errc::invalid_argument;
RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ;
RsGxsServiceTurtleSearchReqItem searchItem(serviceType);
searchItem.mSearchDataSize = dataSize;
searchItem.mSearchData = searchData;
RsGxsNetTunnelTurtleSearchSubstringItem *substring_sr = dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem *>(item) ;
RsGxsNetTunnelSerializer tSerializer;
if(substring_sr != NULL)
{
GXS_NET_TUNNEL_DEBUG() << " : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl;
uint32_t size = tSerializer.size(&searchItem);
uint8_t* buf = rs_malloc<uint8_t>(size);
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ;
tSerializer.serialise(&searchItem, buf, &size);
std::list<RsGxsGroupSummary> results ;
RsNetworkExchangeService *service = nullptr;
requestId = mTurtle->turtleSearch(buf, size, this);
if(!requestId) return std::errc::result_out_of_range;
{
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_sr->service) ;
if(it != mSearchableServices.end())
service = it->second;
}
if(service != nullptr && service->search(substring_sr->substring_match,results))
{
RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item ;
GXS_NET_TUNNEL_DEBUG() << " : " << results.size() << " result found. Sending back." << std::endl;
search_result_item.service = substring_sr->service ;
search_result_item.group_infos = results ;
search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
delete item;
if(search_result_data == NULL)
return false ;
RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
return true ;
}
}
RsGxsNetTunnelTurtleSearchGroupRequestItem *substring_gr = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem *>(item) ;
if(substring_gr != NULL)
{
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(substring_gr->service) ;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA ;
unsigned char *encrypted_group_data = NULL ;
uint32_t encrypted_group_data_len = 0 ;
if(it != mSearchableServices.end() && it->second->search(substring_gr->hashed_group_id,encrypted_group_data,encrypted_group_data_len))
{
RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item ;
search_result_item.service = substring_gr->service ;
search_result_item.encrypted_group_data = encrypted_group_data ;
search_result_item.encrypted_group_data_len = encrypted_group_data_len;
search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ;
search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ;
if(search_result_data == NULL)
return false ;
RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size);
delete item;
return true ;
}
}
delete item;
return false ;
return std::error_condition();
}
void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len)
rs_view_ptr<RsNetworkExchangeService>
RsGxsNetTunnelService::retrievieSearchableServiceLocking(uint16_t serviceType)
{
RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len);
RS_STACK_MUTEX(mGxsNetTunnelMtx);
auto it = mSearchableServices.find(serviceType);
if( it != mSearchableServices.end()) return it->second;
return nullptr;
}
GXS_NET_TUNNEL_DEBUG() << " : received search result for search request " << std::hex << request_id << "" << std::endl;
bool RsGxsNetTunnelService::receiveSearchRequest(
uint8_t* search_request_data, uint32_t search_request_data_len,
uint8_t*& search_result_data, uint32_t& search_result_data_size,
uint32_t& max_allowed_hits )
{
/* Must return true only if there are matching results available, false in
* all other cases. @see RsTurleClientService */
RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem *>(item) ;
RS_DBG3("");
if(result_gs != NULL)
RsGxsNetTunnelSerializer tSerializer;
std::unique_ptr<RsItem> item;
item.reset(tSerializer.deserialise(
search_request_data, &search_request_data_len ));
if(!item)
{
GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl;
#ifdef DEBUG_RSGXSNETTUNNEL
for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it)
std::cerr << " group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl;
#endif
auto it = mSearchableServices.find(result_gs->service) ;
if(it == mSearchableServices.end())
{
GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is not in the searchable services list." << std::endl;
delete item;
return ;
}
it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ;
delete item;
return ;
RS_ERR( "Deserialization failed: ",
search_request_data, search_request_data_len, item.get() );
print_stacktrace();
return false;
}
RsGxsNetTunnelTurtleSearchGroupDataItem *result_gd = dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem *>(item) ;
if(result_gd != NULL)
{
GXS_NET_TUNNEL_DEBUG() << " : result is of type group data for service " << result_gd->service << std::dec << ": " << std::endl;
auto it = mSearchableServices.find(result_gd->service) ;
if(it == mSearchableServices.end())
switch(static_cast<RsGxsNetTunnelItemSubtypes>(item->PacketSubType()))
{
case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING:
{
if(!search_result_data)
{
GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gd->service << std::dec << " that is not in the searchable services list." << std::endl;
delete item;
return ;
RS_ERR( "Got item with TURTLE_SEARCH_SUBSTRING without space for "
"results!" );
print_stacktrace();
break;
}
it->second->receiveTurtleSearchResults(request_id,result_gd->encrypted_group_data,result_gd->encrypted_group_data_len) ;
auto substring_sr =
dynamic_cast<RsGxsNetTunnelTurtleSearchSubstringItem*>(item.get());
if(!substring_sr)
{
RS_WARN( "Got item with TURTLE_SEARCH_SUBSTRING subtype: ",
item->PacketSubType(), " but casting failed!");
break;
}
result_gd->encrypted_group_data = NULL ; // prevents deletion
delete item;
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
std::list<RsGxsGroupSummary> results;
auto tService = retrievieSearchableServiceLocking(substring_sr->service);
if(tService && tService->search(substring_sr->substring_match, results))
{
RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item;
search_result_item.service = substring_sr->service;
search_result_item.group_infos = results;
search_result_data_size = tSerializer.size(&search_result_item);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
return ;
}
tSerializer.serialise(
&search_result_item, search_result_data,
&search_result_data_size );
GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl;
return true;
}
break;
}
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST:
{
auto *substring_gr =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupRequestItem*>(item.get());
if(!substring_gr)
{
RS_WARN( "Got item with TURTLE_SEARCH_GROUP_REQUEST subtype: ",
item->PacketSubType(), " but casting failed!" );
break;
}
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA;
uint8_t* encrypted_group_data = nullptr;
uint32_t encrypted_group_data_len = 0;
auto tService = retrievieSearchableServiceLocking(substring_gr->service);
if(tService && tService->search(
substring_gr->hashed_group_id,
encrypted_group_data, encrypted_group_data_len ))
{
RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item;
search_result_item.service = substring_gr->service;
search_result_item.encrypted_group_data = encrypted_group_data;
search_result_item.encrypted_group_data_len = encrypted_group_data_len;
search_result_data_size = tSerializer.size(&search_result_item);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
tSerializer.serialise(
&search_result_item,
search_result_data, &search_result_data_size );
return true;
}
break;
}
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST:
{
RS_DBG3("SERVICE_SEARCH_REQUEST");
auto searchItem =
static_cast<RsGxsServiceTurtleSearchReqItem*>(item.get());
max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH;
uint16_t sType = static_cast<uint16_t>(searchItem->mServiceType);
auto sService = retrievieSearchableServiceLocking(sType);
if(!sService)
{
RS_WARN("Got search request for non searchable service: ", sType);
break;
}
RsGxsServiceTurtleSearchReplyItem replyItem(searchItem->mServiceType);
auto errc = sService->handleDistantSearchRequest(
searchItem->mSearchData, searchItem->mSearchDataSize,
replyItem.mReplyData, replyItem.mReplyDataSize );
if(errc)
{
// Some error has been reported by the searchable service
RS_WARN("searchable service: ", sType , " reported: ", errc);
break;
}
if( (!replyItem.mReplyData && replyItem.mReplyDataSize) ||
(replyItem.mReplyData && !replyItem.mReplyDataSize) )
{
// Inconsistent behaviour from searcheable service
RS_ERR( "searchable service: ", sType , " silently failed handling "
"inconsistent result mReplyData: ", replyItem.mReplyData,
" mReplyDataSize: ", replyItem.mReplyDataSize );
break;
}
/* Our node have 0 matching results */
if(!replyItem.mReplyData && !replyItem.mReplyDataSize)
break;
search_result_data_size = tSerializer.size(&replyItem);
search_result_data = rs_malloc<uint8_t>(search_result_data_size);
tSerializer.serialise(
&replyItem, search_result_data, &search_result_data_size );
return true;
}
default:
RS_WARN("Got unknown item type: ", item->PacketSubType());
break;
}
return false;
}
void RsGxsNetTunnelService::getStatistics(std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups, std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers, std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId> &turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
void RsGxsNetTunnelService::receiveSearchResult(
TurtleSearchRequestId request_id,
uint8_t* search_result_data, uint32_t search_result_data_len )
{
RS_DBG3(request_id);
std::unique_ptr<RsItem> item;
item.reset(RsGxsNetTunnelSerializer().deserialise(
search_result_data,&search_result_data_len ));
auto castFailedWarn = [](const uint8_t subtype)
{
RS_WARN( "Got item with subtype: ", subtype,
" but cast failed!" );
};
auto searchableServiceGet = [this](const auto pservice)
{
auto service = static_cast<uint16_t>(pservice);
auto it = mSearchableServices.find(service);
if(it == mSearchableServices.end())
{
RS_WARN( "got item for service ", service,
" which is not in the searchable services list." );
return static_cast<RsNetworkExchangeService*>(nullptr);
}
return it->second;
};
const auto tSubtype = item->PacketSubType();
switch (static_cast<RsGxsNetTunnelItemSubtypes>(tSubtype))
{
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY:
{
auto result_gs =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupSummaryItem*>(
item.get() );
if(!result_gs)
{
castFailedWarn(tSubtype);
break;
}
RS_DBG2( " got result is of type group summary result for service ",
result_gs->service );
auto service = searchableServiceGet(result_gs->service);
if(service)
service->receiveTurtleSearchResults(
request_id, result_gs->group_infos );
return;
}
case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA:
{
auto result_gd =
dynamic_cast<RsGxsNetTunnelTurtleSearchGroupDataItem*>(item.get());
if(!result_gd)
{
castFailedWarn(tSubtype);
break;
}
RS_DBG2("got group data result for service: ", result_gd->service);
auto service = searchableServiceGet(result_gd->service);
if(service)
service->receiveTurtleSearchResults(
request_id,
result_gd->encrypted_group_data,
result_gd->encrypted_group_data_len );
/* Ensure ownershipt is passed down preventing deletion */
result_gd->encrypted_group_data = nullptr;
break;
}
case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY:
{
auto searchReply =
static_cast<RsGxsServiceTurtleSearchReplyItem*>(item.get());
auto service = searchableServiceGet(searchReply->mServiceType);
if(service)
service->receiveDistantSearchResult(
request_id,
searchReply->mReplyData,
searchReply->mReplyDataSize );
/* Ensure memory ownership is passed down preventing deletion */
searchReply->mReplyData = nullptr;
break;
}
default:
RS_WARN("got item of unknown type: ", item->PacketSubType());
break;
}
}
void RsGxsNetTunnelService::getStatistics(
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups,
std::map<RsGxsNetTunnelVirtualPeerId,
RsGxsNetTunnelVirtualPeerInfo>& virtual_peers,
std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId>&
turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const
{
groups = mGroups ;
virtual_peers = mVirtualPeers ;

62
libretroshare/src/gxs/rsgxsnettunnel.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2018 Cyril Soler <retroshare.project@gmail.com> *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -23,6 +25,7 @@
#pragma once
#include <map>
#include <system_error>
#include "turtle/p3turtle.h"
#include "retroshare/rsgxsdistsync.h"
@ -100,7 +103,7 @@
// and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen.
//
class RsGxsNetTunnelItem ;
struct RsGxsNetTunnelItem;
class RsNetworkExchangeService ;
class RsGxsNetTunnelService:
@ -108,8 +111,8 @@ class RsGxsNetTunnelService:
public RsGxsDistSync
{
public:
RsGxsNetTunnelService() ;
virtual ~RsGxsNetTunnelService() ;
RsGxsNetTunnelService();
~RsGxsNetTunnelService() override;
/*!
* \brief registerSearchableService
@ -181,24 +184,38 @@ public:
*/
void dump() const;
/*!
* \brief connectToTurtleRouter
* Should be called after allocating a RsGxsNetTunnelService
* \param tr turtle router object
*/
virtual void connectToTurtleRouter(p3turtle *tr) ;
/*!
* Should be called after allocating a RsGxsNetTunnelService
* \param tr turtle router object
*/
void connectToTurtleRouter(p3turtle *tr) override;
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ;
TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ;
/** Gxs services (channels, forums...) are supposed to use this to request
* searches on distant peers */
std::error_condition turtleSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId );
/*!
* \brief receiveSearchRequest
* See RsTurtleClientService::@
*/
virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits);
virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len);
///@see RsTurtleClientService
bool receiveSearchRequest(
unsigned char* search_request_data,
uint32_t search_request_data_len,
unsigned char*& search_result_data,
uint32_t& search_result_data_len,
uint32_t& max_allowed_hits ) override;
void threadTick() override; /// @see RsTickingThread
///@see RsTurtleClientService
virtual void receiveSearchResult(
TurtleSearchRequestId request_id,
unsigned char* search_result_data,
uint32_t search_result_data_len ) override;
TurtleRequestId turtleGroupRequest(
const RsGxsGroupId& group_id,
RsNetworkExchangeService* client_service );
/// @see RsTickingThread
void threadTick() override;
// Overloads p3Config
@ -213,6 +230,11 @@ public:
std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
Bias20Bytes& bias) const;
RS_DEPRECATED
TurtleRequestId turtleSearchRequest(
const std::string& match_string,
RsNetworkExchangeService* client_service );
protected:
// interaction with turtle router
@ -233,6 +255,8 @@ private:
void sendKeepAlivePackets() ;
void handleIncoming(RsGxsNetTunnelItem *item) ;
void flush_pending_items();
rs_view_ptr<RsNetworkExchangeService> retrievieSearchableServiceLocking(
uint16_t serviceType );
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> mGroups ; // groups on the client and server side

10
libretroshare/src/gxs/rsgxsnotify.h
View File

@ -97,3 +97,13 @@ private:
bool mMetaChange;
};
struct RsGxsMsgDeletedChange : RsGxsNotify
{
RsGxsMsgDeletedChange(
const RsGxsGroupId& gid, const RsGxsMessageId& msgId):
RsGxsNotify(gid), messageId(msgId) {}
NotifyType getType() override { return TYPE_MESSAGE_DELETED; }
const RsGxsMessageId messageId;
};

145
libretroshare/src/gxs/rsgxsutil.cc
View File

@ -3,8 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2013-2013 by Christopher Evi-Parker *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2013 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -29,12 +29,6 @@
#include "pqi/pqihash.h"
#include "gxs/rsgixs.h"
#ifdef RS_DEEP_CHANNEL_INDEX
# include "deep_search/channelsindex.hpp"
# include "services/p3gxschannels.h"
# include "rsitems/rsgxschannelitems.h"
#endif
// The goals of this set of methods is to check GXS messages and groups for consistency, mostly
// re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't
// happen anyway, but we still conduct these test as an extra safety measure.
@ -197,9 +191,8 @@ bool RsGxsCleanUp::clean(RsGxsGroupId& next_group_to_check,std::vector<RsGxsGrou
}
RsGxsIntegrityCheck::RsGxsIntegrityCheck(
RsGeneralDataService* const dataService, RsGenExchange* genex,
RsSerialType&
, RsGixs* gixs )
RsGeneralDataService* const dataService, RsGenExchange* genex,
RsSerialType&, RsGixs* gixs )
: mDs(dataService), mGenExchangeClient(genex),
mDone(false), mIntegrityMutex("integrity"), mGixs(gixs) {}
@ -346,19 +339,13 @@ bool RsGxsIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralD
return true;
}
bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
#ifdef RS_DEEP_CHANNEL_INDEX
, RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
#endif
, std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel)
bool RsGxsSinglePassIntegrityCheck::check(
uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel )
{
#ifdef DEBUG_GXSUTIL
GXSUTIL_DEBUG() << "Parsing all groups and messages data in service " << std::hex << mds->serviceType() << " for integrity check. Could take a while..." << std::endl;
#endif
#ifdef RS_DEEP_CHANNEL_INDEX
bool isGxsChannels = mGenExchangeClient->serviceType() == RS_SERVICE_GXS_TYPE_CHANNELS;
std::set<RsGxsGroupId> indexedGroups;
#endif
// first take out all the groups
std::map<RsGxsGroupId, RsNxsGrp*> grp;
@ -393,55 +380,14 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
}
else
msgIds.erase(msgIds.find(grp->grpId)); // could not get them, so group is removed from list.
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels
&& grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC
&& grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )
{
RsGxsGrpMetaData meta;
meta.deserialise(grp->meta.bin_data, grp->meta.bin_len);
uint32_t blz = grp->grp.bin_len;
RsItem* rIt = mSerializer.deserialise(grp->grp.bin_data,
&blz);
if( RsGxsChannelGroupItem* cgIt =
dynamic_cast<RsGxsChannelGroupItem*>(rIt) )
{
RsGxsChannelGroup cg;
cgIt->toChannelGroup(cg, false);
cg.mMeta = meta;
indexedGroups.insert(grp->grpId);
DeepChannelsIndex::indexChannelGroup(cg);
}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Group: "
<< meta.mGroupId.toStdString() << " "
<< meta.mGroupName
<< " doesn't seems a channel, please "
<< "report to developers"
<< std::endl;
print_stacktrace();
}
delete rIt;
}
#endif // def RS_DEEP_CHANNEL_INDEX
}
else
{
std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting group " << grp->grpId << " with wrong hash or null/corrupted meta data. meta=" << grp->metaData << std::endl;
grpsToDel.push_back(grp->grpId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelFromIndex(grp->grpId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
else
{
RS_WARN( "deleting group ", grp->grpId,
" with wrong hash or null/corrupted meta data. meta=",
grp->metaData );
grpsToDel.push_back(grp->grpId);
}
delete grp;
}
@ -469,15 +415,9 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
if(nxsMsg)
nxsMsgS.insert(nxsMsg->msgId);
for (auto& msgId:msgIdV)
if(nxsMsgS.find(msgId) == nxsMsgS.end())
{
msgsToDel[grpId].insert(msgId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelPostFromIndex(grpId, msgId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
for (auto& msgId:msgIdV)
if(nxsMsgS.find(msgId) == nxsMsgS.end())
msgsToDel[grpId].insert(msgId);
}
for(auto mit = msgs.begin(); mit != msgs.end(); ++mit)
@ -495,54 +435,11 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs,
if(msg->metaData == NULL || currHash != msg->metaData->mHash)
{
std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting message " << msg->msgId << " in group " << msg->grpId << " with wrong hash or null/corrupted meta data. meta=" << (void*)msg->metaData << std::endl;
RS_WARN( "deleting message ", msg->msgId, " in group ",
msg->grpId,
" with wrong hash or null/corrupted meta data. meta=",
static_cast<void*>(msg->metaData) );
msgsToDel[msg->grpId].insert(msg->msgId);
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelPostFromIndex(
msg->grpId, msg->msgId );
#endif // def RS_DEEP_CHANNEL_INDEX
}
else if (subscribed_groups.count(msg->metaData->mGroupId))
{
#ifdef RS_DEEP_CHANNEL_INDEX
// This should be moved to p3gxschannels. It is really not the place for this here!
if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) )
{
RsGxsMsgMetaData meta;
meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len);
uint32_t blz = msg->msg.bin_len;
RsItem* rIt = mSerializer.deserialise(msg->msg.bin_data,
&blz);
if( RsGxsChannelPostItem* cgIt =
dynamic_cast<RsGxsChannelPostItem*>(rIt) )
{
RsGxsChannelPost cg;
cgIt->toChannelPost(cg, false);
cg.mMeta = meta;
DeepChannelsIndex::indexChannelPost(cg);
}
else if(dynamic_cast<RsGxsCommentItem*>(rIt)) {}
else if(dynamic_cast<RsGxsVoteItem*>(rIt)) {}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Message: "
<< meta.mMsgId.toStdString()
<< " in group: "
<< meta.mGroupId.toStdString() << " "
<< " doesn't seems a channel post, please "
<< "report to developers"
<< std::endl;
print_stacktrace();
}
delete rIt;
}
#endif // def RS_DEEP_CHANNEL_INDEX
}
delete msg;

36
libretroshare/src/gxs/rsgxsutil.h
View File

@ -3,8 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2013-2013 by Christopher Evi-Parker *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2013 Christopher Evi-Parker *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -162,18 +163,9 @@ class RsGxsIntegrityCheck : public RsThread
enum CheckState { CheckStart, CheckChecking };
public:
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
RsGxsIntegrityCheck(RsGeneralDataService* const dataService,
RsGenExchange *genex, RsSerialType&,
RsGixs *gixs);
RsGxsIntegrityCheck( RsGeneralDataService* const dataService,
RsGenExchange* genex, RsSerialType&,
RsGixs* gixs );
static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds);
bool isDone();
@ -201,19 +193,9 @@ private:
class RsGxsSinglePassIntegrityCheck
{
public:
/*!
*
* @param dataService
* @param mGroupTS
* @param chunkSize
* @param sleepPeriod
*/
static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds
#ifdef RS_DEEP_CHANNEL_INDEX
, RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer
#endif
, std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel);
static bool check(
uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds,
std::vector<RsGxsGroupId>& grpsToDel, GxsMsgReq& msgsToDel );
};
class GroupUpdate

62
libretroshare/src/gxs/rsnxs.h
View File

@ -3,8 +3,10 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2011-2011 by Robert Fernie <retroshare.project@gmail.com> *
* Copyright 2011-2011 by Christopher Evi-Parker *
* Copyright (C) 2011 Robert Fernie <retroshare.project@gmail.com> *
* Copyright (C) 2011 Christopher Evi-Parker *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -20,17 +22,15 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSGNP_H
#define RSGNP_H
#pragma once
#include <set>
#include <string>
#include "util/rstime.h"
#include <stdlib.h>
#include <list>
#include <map>
#include "util/rstime.h"
#include "services/p3service.h"
#include "retroshare/rsreputations.h"
#include "retroshare/rsidentity.h"
@ -61,9 +61,8 @@
class RsNetworkExchangeService
{
public:
RsNetworkExchangeService(){ return;}
virtual ~RsNetworkExchangeService() {}
RsNetworkExchangeService() = default;
virtual ~RsNetworkExchangeService() = default;
virtual uint16_t serviceType() const =0;
/*!
@ -85,9 +84,24 @@ public:
virtual bool msgAutoSync() const =0;
virtual bool grpAutoSync() const =0;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS ///
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS ///
////////////////////////////////////////////////////////////////////////////
/// Trigger remote generic GXS service search
virtual std::error_condition distantSearchRequest(
rs_owner_ptr<uint8_t> searchData, uint32_t dataSize,
RsServiceType serviceType, TurtleRequestId& requestId ) = 0;
/// Handle remote generic GXS services search requests to specific service
virtual std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
/// Receive remote generic GXS services search result
virtual std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) = 0;
/*!
* \brief turtleGroupRequest
@ -115,13 +129,17 @@ public:
*/
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)=0;
/*!
* \brief receiveTurtleSearchResults
* Called by turtle (through RsGxsNetTunnel) when new data is received
* \param req Turtle search request ID associated with this result
* \param encrypted_group_data Group data
*/
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0;
/*!
* \brief receiveTurtleSearchResults
* Called by turtle (through RsGxsNetTunnel) when new data is received
* \param req Turtle search request ID associated with this result
* \param encrypted_group_data Group data
*/
RS_DEPRECATED_FOR("receiveDistantSearchResult")
virtual void receiveTurtleSearchResults(
TurtleRequestId req,
rs_owner_ptr<const uint8_t> encrypted_group_data,
uint32_t encrypted_group_data_len ) = 0;
/*!
* \brief retrieveTurtleSearchResults
@ -141,7 +159,9 @@ public:
virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0;
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
RS_DEPRECATED_FOR("handleDistantSearchRequest and distantSearchRequest")
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;
/*!
@ -306,5 +326,3 @@ public:
}
}
};
#endif // RSGNP_H

60
libretroshare/src/util/rserrno.h → libretroshare/src/gxs/rsnxsobserver.cpp
View File

@ -1,24 +1,36 @@
/*******************************************************************************
* libretroshare/src/util: rserrno.h *
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
const char* rsErrnoName(int err);
/*******************************************************************************
* RetroShare General eXchange System *
* *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "gxs/rsnxsobserver.h"
const RsNxsObserverErrorCategory RsNxsObserverErrorCategory::instance;
std::error_condition RsNxsObserverErrorCategory::default_error_condition(int ev)
const noexcept
{
switch(static_cast<RsNxsObserverErrorNum>(ev))
{
case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
return std::errc::operation_not_supported;
default:
return std::error_condition(ev, *this);
}
}

108
libretroshare/src/gxs/rsnxsobserver.h
View File

@ -3,7 +3,10 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2011-2012 by Robert Fernie, Evi-Parker Christopher *
* Copyright (C) 2011-2012 Robert Fernie *
* Copyright (C) 2011-2012 Christopher Evi-Parker *
* Copyright (C) 2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,21 +22,61 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RSNXSOBSERVER_H
#define RSNXSOBSERVER_H
#pragma once
#include <set>
#include <system_error>
#include <vector>
#include "retroshare/rsgxsiface.h"
#include "rsitems/rsnxsitems.h"
#include "util/rsdebug.h"
typedef uint32_t TurtleRequestId ;
typedef uint32_t TurtleRequestId;
enum class RsNxsObserverErrorNum : int32_t
{
NOT_OVERRIDDEN_BY_OBSERVER = 2004,
};
struct RsNxsObserverErrorCategory: std::error_category
{
const char* name() const noexcept override
{ return "RetroShare NXS Observer"; }
std::string message(int ev) const override
{
switch (static_cast<RsNxsObserverErrorNum>(ev))
{
case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER:
return "Method not overridden by observer";
default:
return rsErrorNotInCategory(ev, name());
}
}
std::error_condition default_error_condition(int ev) const noexcept override;
const static RsNxsObserverErrorCategory instance;
};
namespace std
{
/** Register RsNxsObserverErrorNum as an error condition enum, must be in std
* namespace */
template<> struct is_error_condition_enum<RsNxsObserverErrorNum> : true_type {};
}
/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in
* same namespace of RsJsonApiErrorNum */
inline std::error_condition make_error_condition(RsNxsObserverErrorNum e) noexcept
{
return std::error_condition(
static_cast<int>(e), RsNxsObserverErrorCategory::instance );
};
class RsNxsObserver
{
public:
RsNxsObserver() {}
public:
/*!
@ -56,6 +99,46 @@ public:
std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl;
}
/** If advanced search functionalities like deep indexing are supported at
* observer/service level, this method should be overridden to handle search
* requests there.
* @param[in] requestData search query
* @param[in] requestSize search query size
* @param[out] resultData results data storage for a pointer to search
* result reply data or nullptr if no mathing results where found
* @param[out] resultSize storage for results data size or 0 if no matching
* results where found
* @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
* returned to inform the caller that this method was not overridden by the
* observer so do not use it for other meanings. */
virtual std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
/* Avoid unused paramethers warning this way so doxygen can still parse
* paramethers documentation */
(void) requestData; (void) requestSize;
(void) resultData; (void) resultSize;
return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
}
/** If advanced search functionalities like deep indexing are supported at
* observer/service level, this method should be overridden to handle search
* results there.
* @param[in] requestId search query id
* @param[out] resultData results data
* @param[out] resultSize results data size
* @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is
* returned to inform the caller that this method was not overridden by the
* observer so do not use it for other meanings. */
virtual std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
(void) requestId; (void) resultData; (void) resultSize;
return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER;
}
/*!
* @param grpId group id
*/
@ -70,6 +153,7 @@ public:
* @param grpId group id
*/
virtual void notifyChangedGroupStats(const RsGxsGroupId &grpId) = 0;
};
#endif // RSNXSOBSERVER_H
RsNxsObserver() = default;
virtual ~RsNxsObserver() = default;
};

64
libretroshare/src/libretroshare.pro
View File

@ -157,7 +157,7 @@ rs_webui {
HEADERS += plugins/pluginmanager.h \
plugins/dlfcn_win32.h \
rsitems/rspluginitems.h \
rsitems/rspluginitems.h \
util/i2pcommon.h \
util/rsinitedptr.h
@ -394,7 +394,6 @@ HEADERS += pqi/authssl.h \
pqi/pqissl.h \
pqi/pqissllistener.h \
pqi/pqisslpersongrp.h \
pqi/pqissli2pbob.h \
pqi/pqisslproxy.h \
pqi/pqistore.h \
pqi/pqistreamer.h \
@ -460,7 +459,7 @@ HEADERS += rsitems/rsitem.h \
rsitems/rsgxsupdateitems.h \
rsitems/rsserviceinfoitems.h \
HEADERS += services/autoproxy/p3i2pbob.h \
HEADERS += \
services/rseventsservice.h \
services/autoproxy/rsautoproxymonitor.h \
services/p3msgservice.h \
@ -514,8 +513,7 @@ HEADERS += util/folderiterator.h \
util/cxx11retrocompat.h \
util/cxx14retrocompat.h \
util/cxx17retrocompat.h \
util/rsurl.h \
util/rserrno.h
util/rsurl.h
SOURCES += ft/ftchunkmap.cc \
ft/ftcontroller.cc \
@ -564,7 +562,6 @@ SOURCES += pqi/authgpg.cc \
pqi/pqissl.cc \
pqi/pqissllistener.cc \
pqi/pqisslpersongrp.cc \
pqi/pqissli2pbob.cpp \
pqi/pqisslproxy.cc \
pqi/pqistore.cc \
pqi/pqistreamer.cc \
@ -626,7 +623,6 @@ SOURCES += serialiser/rsbaseserial.cc \
SOURCES += services/autoproxy/rsautoproxymonitor.cc \
services/rseventsservice.cc \
services/autoproxy/p3i2pbob.cc \
services/p3msgservice.cc \
services/p3service.cc \
services/p3statusservice.cc \
@ -646,10 +642,10 @@ SOURCES += util/folderiterator.cc \
util/rsexpr.cc \
util/smallobject.cc \
util/rsdir.cc \
util/rsmemory.cc \
util/rsdiscspace.cc \
util/rsnet.cc \
util/rsnet_ss.cc \
util/rsdnsutils.cc \
util/extaddrfinder.cc \
util/dnsresolver.cc \
util/rsprint.cc \
@ -660,8 +656,7 @@ SOURCES += util/folderiterator.cc \
util/rsrecogn.cc \
util/rstime.cc \
util/rsurl.cc \
util/rsbase64.cc \
util/rserrno.cc
util/rsbase64.cc
equals(RS_UPNP_LIB, miniupnpc) {
HEADERS += rs_upnp/upnputil.h rs_upnp/upnphandler_miniupnp.h
@ -725,7 +720,8 @@ SOURCES += rsitems/rsnxsitems.cc \
gxs/gxstokenqueue.cc \
gxs/rsgxsnetutils.cc \
gxs/rsgxsutil.cc \
gxs/rsgxsrequesttypes.cc
gxs/rsgxsrequesttypes.cc \
gxs/rsnxsobserver.cpp
# gxs tunnels
HEADERS += gxstunnel/p3gxstunnel.h \
@ -949,6 +945,14 @@ rs_jsonapi {
SOURCES += jsonapi/jsonapi.cpp
}
rs_deep_forums_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp
HEADERS += deep_search/forumsindex.hpp
SOURCES += deep_search/forumsindex.cpp
}
rs_deep_channels_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp
@ -1022,6 +1026,44 @@ rs_broadcast_discovery {
}
}
rs_sam3 {
SOURCES += \
services/autoproxy/p3i2psam3.cpp \
pqi/pqissli2psam3.cpp \
HEADERS += \
services/autoproxy/p3i2psam3.h \
pqi/pqissli2psam3.h \
}
rs_sam3_libsam3 {
DUMMYQMAKECOMPILERINPUT = FORCE
libsam3.name = Generating libsam3.
libsam3.input = DUMMYQMAKECOMPILERINPUT
libsam3.output = $$clean_path($${LIBSAM3_BUILD_PATH}/libsam3.a)
libsam3.CONFIG += target_predeps combine
libsam3.variable_out = PRE_TARGETDEPS
win32-g++:isEmpty(QMAKE_SH) {
LIBSAM3_MAKE_PARAMS = CC=gcc
libsam3.commands = \
cd /D $$shell_path($${RS_SRC_PATH}) && git submodule update --init supportlibs/libsam3 || cd . $$escape_expand(\\n\\t) \
$(CHK_DIR_EXISTS) $$shell_path($$LIBSAM3_BUILD_PATH) $(MKDIR) $$shell_path($${LIBSAM3_BUILD_PATH}) $$escape_expand(\\n\\t) \
$(COPY_DIR) $$shell_path($${LIBSAM3_SRC_PATH}) $$shell_path($${LIBSAM3_BUILD_PATH}) || cd . $$escape_expand(\\n\\t)
} else {
LIBSAM3_MAKE_PARAMS =
libsam3.commands = \
cd $${RS_SRC_PATH} && ( \
git submodule update --init supportlibs/libsam3 || \
true ) && \
mkdir -p $${LIBSAM3_BUILD_PATH} && \
(cp -r $${LIBSAM3_SRC_PATH}/* $${LIBSAM3_BUILD_PATH} || true) &&
}
libsam3.commands += \
cd $$shell_path($${LIBSAM3_BUILD_PATH}) && \
$(MAKE) build $${LIBSAM3_MAKE_PARAMS}
QMAKE_EXTRA_COMPILERS += libsam3
}
###########################################################################################################
# OLD CONFIG OPTIONS.
# Not used much - but might be useful one day.

244
libretroshare/src/pqi/p3netmgr.cc
View File

@ -47,14 +47,15 @@ struct RsLog::logInfo p3netmgrzoneInfo = {RsLog::Default, "p3netmgr"};
/* Network setup States */
const uint32_t RS_NET_NEEDS_RESET = 0x0000;
const uint32_t RS_NET_UNKNOWN = 0x0001;
const uint32_t RS_NET_UPNP_INIT = 0x0002;
const uint32_t RS_NET_UPNP_SETUP = 0x0003;
const uint32_t RS_NET_EXT_SETUP = 0x0004;
const uint32_t RS_NET_DONE = 0x0005;
const uint32_t RS_NET_LOOPBACK = 0x0006;
//const uint32_t RS_NET_DOWN = 0x0007;
constexpr uint32_t RS_NET_NEEDS_RESET = 0x0000;
constexpr uint32_t RS_NET_UNKNOWN = 0x0001;
constexpr uint32_t RS_NET_UPNP_INIT = 0x0002;
constexpr uint32_t RS_NET_UPNP_SETUP = 0x0003;
constexpr uint32_t RS_NET_EXT_SETUP = 0x0004;
constexpr uint32_t RS_NET_DONE = 0x0005;
constexpr uint32_t RS_NET_LOOPBACK = 0x0006;
//constexpr uint32_t RS_NET_DOWN = 0x0007;
constexpr uint32_t RS_NET_SHUTDOWN = 0x00FF; //Highest value to not restart UPnP nor ExtAddrFinder
/* Stun modes (TODO) */
//const uint32_t RS_STUN_DHT = 0x0001;
@ -68,6 +69,8 @@ const uint32_t MAX_UPNP_COMPLETE = 600; /* 10 min... seems to take a while */
//const uint32_t MIN_TIME_BETWEEN_NET_RESET = 5;
const uint32_t MIN_TIME_EXT_FINDER_UPDATE = 300; /* 5min to check if external IP is changed */
/****
* #define NETMGR_DEBUG 1
* #define NETMGR_DEBUG_RESET 1
@ -107,19 +110,17 @@ void pqiNetStatus::print(std::ostream &out)
}
p3NetMgrIMPL::p3NetMgrIMPL() : mPeerMgr(nullptr), mLinkMgr(nullptr),
mNetMtx("p3NetMgr"), mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false),
mDoNotNetCheckUntilTs(0)
p3NetMgrIMPL::p3NetMgrIMPL()
: mPeerMgr(nullptr), mLinkMgr(nullptr)
, mNetMtx("p3NetMgr"), mNetMode(RS_NET_MODE_UDP), mVsDisc(RS_VS_DISC_FULL), mVsDht(RS_VS_DHT_FULL)// default to full.
, mNetInitTS(0), mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false)
, mUseExtAddrFinder(true), mNetExtAddrFinderTs(0), mDoNotNetCheckUntilTs(0)
{
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetMode = RS_NET_MODE_UDP;
mUseExtAddrFinder = true;
mExtAddrFinder = new ExtAddrFinder();
mNetInitTS = 0;
mNetFlags = pqiNetStatus();
mOldNetFlags = pqiNetStatus();
@ -133,10 +134,6 @@ p3NetMgrIMPL::p3NetMgrIMPL() : mPeerMgr(nullptr), mLinkMgr(nullptr),
mLocalAddr.ss_family = AF_INET;
mExtAddr.ss_family = AF_INET;
// default to full.
mVsDisc = RS_VS_DISC_FULL;
mVsDht = RS_VS_DHT_FULL;
}
#ifdef NETMGR_DEBUG
@ -235,6 +232,10 @@ void p3NetMgrIMPL::netReset()
rslog(RSL_ALERT, p3netmgrzone, "p3NetMgr::netReset() Called");
shutdown(); /* blocking shutdown call */
{
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStatus = RS_NET_UNKNOWN;
}
// Will initiate a new call for determining the external ip.
if (mUseExtAddrFinder)
@ -242,7 +243,7 @@ void p3NetMgrIMPL::netReset()
#ifdef NETMGR_DEBUG_RESET
std::cerr << "p3NetMgrIMPL::netReset() restarting AddrFinder" << std::endl;
#endif
mExtAddrFinder->reset() ;
mExtAddrFinder->reset(true) ;
}
else
{
@ -265,7 +266,7 @@ void p3NetMgrIMPL::netReset()
* as it calls back to p3ConnMgr.
*/
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
struct sockaddr_storage iaddr = mLocalAddr;
@ -283,8 +284,7 @@ void p3NetMgrIMPL::netReset()
}
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStatus = RS_NET_UNKNOWN;
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
netStatusReset_locked();
}
@ -311,8 +311,8 @@ bool p3NetMgrIMPL::shutdown() /* blocking shutdown call */
std::cerr << std::endl;
#endif
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStatus = RS_NET_UNKNOWN;
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStatus = RS_NET_SHUTDOWN;
mNetInitTS = time(NULL);
netStatusReset_locked();
}
@ -347,7 +347,7 @@ void p3NetMgrIMPL::netStartup()
*/
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetInitTS = time(NULL);
netStatusReset_locked();
@ -460,35 +460,51 @@ void p3NetMgrIMPL::netStatusTick()
uint32_t netStatus = 0;
rstime_t age = 0;
bool needExtFinderUpdate = false;
{
RsStackMutex stack(mNetMtx); /************** LOCK MUTEX ***************/
RS_STACK_MUTEX(mNetMtx); /************** LOCK MUTEX ***************/
netStatus = mNetStatus;
age = time(NULL) - mNetInitTS;
}
needExtFinderUpdate = netStatus == RS_NET_DONE;
needExtFinderUpdate &= mNetExtAddrFinderTs < time(nullptr);
if(needExtFinderUpdate)
mNetExtAddrFinderTs = time(nullptr) + MIN_TIME_EXT_FINDER_UPDATE;
}
if(netStatus <= RS_NET_UPNP_SETUP && mUseExtAddrFinder)
{
sockaddr_storage tmpip = mLocalAddr; // copies local port and correctly inits the IP family
if( mUseExtAddrFinder
&& ( netStatus <= RS_NET_UPNP_SETUP
|| needExtFinderUpdate) )
{
sockaddr_storage tmpip;
sockaddr_storage_copy( mLocalAddr, tmpip); // copies local port and correctly inits the IP family
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "Asking ExtAddrFinder for IP. Initializing port with " << sockaddr_storage_port(tmpip) << std::endl;
RS_DBG("Asking ExtAddrFinder for IP. Initializing port with ", sockaddr_storage_port(tmpip));
#endif
if(mExtAddrFinder->hasValidIP(tmpip) && sockaddr_storage_ipv6_to_ipv4(tmpip) && !sockaddr_storage_same(tmpip,mExtAddr))
{
if(mExtAddrFinder->hasValidIPV4(tmpip))
{
if(!sockaddr_storage_same(tmpip,mExtAddr))
{
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() Ext supplied by ExtAddrFinder" << std::endl;
RS_DBG("Ext supplied by ExtAddrFinder. ExtAddr: ", tmpip);
#endif
setExtAddress(tmpip);
}
}
else if(mExtAddrFinder->hasValidIPV6(tmpip))
{
if(!sockaddr_storage_same(tmpip,mExtAddr))
{
//Only if no IPv4 else, reset connections on setExtAddress()
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() ";
std::cerr << "ExtAddr: " << sockaddr_storage_tostring(tmpip);
std::cerr << std::endl;
RS_DBG("Ext supplied by ExtAddrFinder. ExtAddr: ", tmpip);
#endif
setExtAddress(tmpip);
}
}
setExtAddress(tmpip);
}
}
}
switch(netStatus)
{
@ -578,7 +594,7 @@ void p3NetMgrIMPL::netDhtInit()
uint32_t vs = 0;
{
RsStackMutex stack(mNetMtx); /*********** LOCKED MUTEX ************/
RS_STACK_MUTEX(mNetMtx); /*********** LOCKED MUTEX ************/
vs = mVsDht;
}
@ -745,25 +761,16 @@ void p3NetMgrIMPL::netExtCheck()
if (mUseExtAddrFinder)
{
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() checking ExtAddrFinder" << std::endl;
RS_DBG("checking ExtAddrFinder");
#endif
sockaddr_storage tmpip = mLocalAddr; // copies local port and correctly inits the IP family
sockaddr_storage tmpip;
sockaddr_storage_copy( mLocalAddr, tmpip); // copies local port and correctly inits the IP family
bool extFinderOk = mExtAddrFinder->hasValidIP(tmpip);
if (extFinderOk && sockaddr_storage_ipv6_to_ipv4(tmpip))
// Test for IPv4 first to be compatible with older versions.
if (mExtAddrFinder->hasValidIPV4(tmpip))
{
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() Ext supplied by ExtAddrFinder" << std::endl;
#endif
sockaddr_storage_setport(tmpip, guessNewExtPort());
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::netExtCheck() ";
std::cerr << "ExtAddr: " << sockaddr_storage_tostring(tmpip);
std::cerr << std::endl;
#endif
mNetFlags.mExtAddrOk = true;
address_votes[tmpip].n++ ;
@ -773,10 +780,22 @@ void p3NetMgrIMPL::netExtCheck()
* (which it is not normally) */
mNetFlags.mExtAddrStableOk = true;
std::cerr << __PRETTY_FUNCTION__ << " ExtAddrFinder "
<< " reported external address "
<< sockaddr_storage_iptostring(tmpip)
<< std::endl;
RS_DBG("Reported external IPv4 address ", sockaddr_storage_iptostring(tmpip));
}
else if (mExtAddrFinder->hasValidIPV6(tmpip))
{
sockaddr_storage_setport(tmpip, guessNewExtPort());
mNetFlags.mExtAddrOk = true;
address_votes[tmpip].n++ ;
/* XXX HACK TO FIX drbob: ALLOWING
* ExtAddrFinder -> ExtAddrStableOk = true
* (which it is not normally) */
mNetFlags.mExtAddrStableOk = true;
RS_DBG("Reported external IPv6 address ", sockaddr_storage_iptostring(tmpip));
}
}
}
@ -967,7 +986,7 @@ void p3NetMgrIMPL::netExtCheck()
if (netSetupDone)
{
std::cerr << "p3NetMgrIMPL::netExtCheck() netSetupDone" << std::endl;
RS_DBG("netSetupDone");
/* Setup NetStateBox with this info */
updateNetStateBox_startup();
@ -978,12 +997,14 @@ void p3NetMgrIMPL::netExtCheck()
mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
}
/* inform DHT about our external address */
RsPeerId fakeId;
netAssistKnownPeer(fakeId, mExtAddr, NETASSIST_KNOWN_PEER_SELF | NETASSIST_KNOWN_PEER_ONLINE);
/* inform DHT about our external IPV4 address, it doesn't support IPv6 for now.*/
if(sockaddr_storage_ipv6_to_ipv4(mExtAddr))
{
RsPeerId fakeId;
netAssistKnownPeer(fakeId, mExtAddr, NETASSIST_KNOWN_PEER_SELF | NETASSIST_KNOWN_PEER_ONLINE);
}
std::cerr << __PRETTY_FUNCTION__ << " Network Setup Complete"
<< std::endl;
RS_INFO("Network Setup Complete");
}
}
@ -994,7 +1015,8 @@ void p3NetMgrIMPL::netExtCheck()
bool p3NetMgrIMPL::checkNetAddress()
{
bool addrChanged = false;
bool validAddr = false;
bool validAddr = false;
bool needOwnAddrUpdate = false;
sockaddr_storage prefAddr;
sockaddr_storage oldAddr;
@ -1067,8 +1089,7 @@ bool p3NetMgrIMPL::checkNetAddress()
if (!validAddr)
{
RsErr() << __PRETTY_FUNCTION__ << " no valid local network address "
<<" found. Report to developers." << std::endl;
RS_ERR("no valid local network address found. Report to developers.");
print_stacktrace();
return false;
@ -1109,6 +1130,7 @@ bool p3NetMgrIMPL::checkNetAddress()
sockaddr_storage_setport(mExtAddr, port); // this accounts for when the port was updated
addrChanged = true;
}
} // RS_STACK_MUTEX(mNetMtx);
if (addrChanged)
@ -1123,11 +1145,24 @@ bool p3NetMgrIMPL::checkNetAddress()
rsEvents->postEvent(ev);
}
if (mPeerMgr) mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
needOwnAddrUpdate = true;
netReset();
}
if (mPeerMgr)
{
// Retrieve last known IP, if none, update own addresse to get current.
peerState ps;
mPeerMgr->getOwnNetStatus(ps);
needOwnAddrUpdate |= ps.ipAddrs.mLocal.mAddrs.empty();
needOwnAddrUpdate |= ps.ipAddrs.mExt.mAddrs.empty();
if (needOwnAddrUpdate)
{
mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
}
}
return true;
}
@ -1139,7 +1174,7 @@ bool p3NetMgrIMPL::checkNetAddress()
/* to allow resets of network stuff */
void p3NetMgrIMPL::addNetListener(pqiNetListener *listener)
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetListeners.push_back(listener);
}
@ -1149,7 +1184,7 @@ bool p3NetMgrIMPL::setLocalAddress(const struct sockaddr_storage &addr)
{
bool changed = false;
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
if (!sockaddr_storage_same(mLocalAddr, addr))
{
changed = true;
@ -1171,7 +1206,7 @@ bool p3NetMgrIMPL::setLocalAddress(const struct sockaddr_storage &addr)
}
bool p3NetMgrIMPL::getExtAddress(struct sockaddr_storage& addr)
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
if(mNetFlags.mExtAddrOk)
{
@ -1186,7 +1221,7 @@ bool p3NetMgrIMPL::setExtAddress(const struct sockaddr_storage &addr)
{
bool changed = false;
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
if (!sockaddr_storage_same(mExtAddr, addr))
{
changed = true;
@ -1220,7 +1255,7 @@ bool p3NetMgrIMPL::setNetworkMode(uint32_t netMode)
{
uint32_t oldNetMode;
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
/* only change TRY flags */
oldNetMode = mNetMode;
@ -1266,7 +1301,7 @@ bool p3NetMgrIMPL::setNetworkMode(uint32_t netMode)
bool p3NetMgrIMPL::setVisState(uint16_t vs_disc, uint16_t vs_dht)
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mVsDisc = vs_disc;
mVsDht = vs_dht;
@ -1628,7 +1663,7 @@ void p3NetMgrIMPL::getNetStatus(pqiNetStatus &status)
uint32_t netsize = 0, rsnetsize = 0;
netAssistConnectStats(netsize, rsnetsize);
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
/* quick update of the stuff that can change! */
mNetFlags.mDhtOk = dhtOk;
@ -1652,7 +1687,7 @@ void p3NetMgrIMPL::getNetStatus(pqiNetStatus &status)
bool p3NetMgrIMPL::getIPServersEnabled()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mUseExtAddrFinder;
}
@ -1661,15 +1696,32 @@ void p3NetMgrIMPL::getIPServersList(std::list<std::string>& ip_servers)
mExtAddrFinder->getIPServersList(ip_servers);
}
void p3NetMgrIMPL::getCurrentExtIPList(std::list<std::string>& ip_list)
{
ip_list.clear();
sockaddr_storage addr;
if(mExtAddrFinder->hasValidIPV4(addr))
ip_list.push_back(sockaddr_storage_iptostring(addr));
if(mExtAddrFinder->hasValidIPV6(addr))
ip_list.push_back(sockaddr_storage_iptostring(addr));
}
void p3NetMgrIMPL::setIPServersEnabled(bool b)
{
if (mUseExtAddrFinder != b)
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
mExtAddrFinder->reset(true);
if (b)
mExtAddrFinder->start_request();
}
{
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mUseExtAddrFinder = b;
}
#ifdef NETMGR_DEBUG
std::cerr << "p3NetMgr: setIPServers to " << b << std::endl ;
RS_DBG("set mUseExtAddrFinder to ", b);
#endif
}
@ -1682,31 +1734,31 @@ void p3NetMgrIMPL::setIPServersEnabled(bool b)
RsNetState p3NetMgrIMPL::getNetStateMode()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getNetStateMode();
}
RsNetworkMode p3NetMgrIMPL::getNetworkMode()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getNetworkMode();
}
RsNatTypeMode p3NetMgrIMPL::getNatTypeMode()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getNatTypeMode();
}
RsNatHoleMode p3NetMgrIMPL::getNatHoleMode()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getNatHoleMode();
}
RsConnectModes p3NetMgrIMPL::getConnectModes()
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
return mNetStateBox.getConnectModes();
}
@ -1730,7 +1782,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
/* input network bits */
if (mDhtStunner->getExternalAddr(tmpaddr, isstable))
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStateBox.setAddressStunDht(tmpaddr, isstable);
#ifdef NETMGR_DEBUG_STATEBOX
@ -1749,7 +1801,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
/* input network bits */
if (mProxyStunner->getExternalAddr(tmpaddr, isstable))
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStateBox.setAddressStunProxy(tmpaddr, isstable);
#ifdef NETMGR_DEBUG_STATEBOX
@ -1768,7 +1820,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
bool dhtOn = netAssistConnectEnabled();
bool dhtActive = netAssistConnectActive();
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStateBox.setDhtState(dhtOn, dhtActive);
}
@ -1777,7 +1829,7 @@ void p3NetMgrIMPL::updateNetStateBox_temporal()
#ifdef NETMGR_DEBUG_STATEBOX
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
auto netstate = mNetStateBox.getNetStateMode();
auto netMode = mNetStateBox.getNetworkMode();
@ -1823,7 +1875,7 @@ void p3NetMgrIMPL::updateNatSetting()
RsNatTypeMode natType = RsNatTypeMode::UNKNOWN;
RsNatHoleMode natHole = RsNatHoleMode::UNKNOWN;
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
natType = mNetStateBox.getNatTypeMode();
natHole = mNetStateBox.getNatHoleMode();
@ -1925,7 +1977,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
std::cerr << std::endl;
#endif
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
/* fill in the data */
struct sockaddr_storage tmpip;
@ -1973,8 +2025,8 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
/* ExtAddrFinder */
if (mUseExtAddrFinder)
{
tmpip = mLocalAddr;
bool extFinderOk = mExtAddrFinder->hasValidIP(tmpip);
tmpip = mLocalAddr;
bool extFinderOk = mExtAddrFinder->hasValidIPV4(tmpip);
if (extFinderOk)
{
@ -2020,7 +2072,7 @@ void p3NetMgrIMPL::updateNetStateBox_startup()
void p3NetMgrIMPL::updateNetStateBox_reset()
{
{
RsStackMutex stack(mNetMtx); /****** STACK LOCK MUTEX *******/
RS_STACK_MUTEX(mNetMtx); /****** STACK LOCK MUTEX *******/
mNetStateBox.reset();

7
libretroshare/src/pqi/p3netmgr.h
View File

@ -120,6 +120,7 @@ virtual bool shutdown() = 0; /* blocking shutdown call */
virtual bool getIPServersEnabled() = 0;
virtual void setIPServersEnabled(bool b) = 0;
virtual void getIPServersList(std::list<std::string>& ip_servers) = 0;
virtual void getCurrentExtIPList(std::list<std::string>& ip_list) = 0;
// ONLY USED by p3face-config.cc WHICH WILL BE REMOVED.
virtual void getNetStatus(pqiNetStatus &status) = 0;
@ -171,6 +172,7 @@ virtual bool shutdown(); /* blocking shutdown call */
virtual bool getIPServersEnabled();
virtual void setIPServersEnabled(bool b);
virtual void getIPServersList(std::list<std::string>& ip_servers);
virtual void getCurrentExtIPList(std::list<std::string>& ip_list);
// ONLY USED by p3face-config.cc WHICH WILL BE REMOVED.
virtual void getNetStatus(pqiNetStatus &status);
@ -307,12 +309,13 @@ void netStatusReset_locked();
uint16_t mVsDisc;
uint16_t mVsDht;
rstime_t mNetInitTS;
rstime_t mNetInitTS;
uint32_t mNetStatus;
bool mStatusChanged;
bool mUseExtAddrFinder;
bool mUseExtAddrFinder;
rstime_t mNetExtAddrFinderTs;
/* network status flags (read by rsiface) */
pqiNetStatus mNetFlags;

7
libretroshare/src/pqi/pqissl.cc
View File

@ -1742,6 +1742,13 @@ bool pqissl::moretoread(uint32_t usec)
{
rslog(RSL_ALERT, pqisslzone,
"pqissl::moretoread() Select ERROR!");
RS_WARN(strerror(errno));
if (errno == EBADF) {
// happens when SAM is shut down
rslog(RSL_ALERT, pqisslzone, "pqissl::moretoread() -> calling reset()");
reset_locked();
}
return 0;
}

52
libretroshare/src/pqi/pqissli2pbob.cpp
View File

@ -1,52 +0,0 @@
/*******************************************************************************
* libretroshare/src/pqi: pqissli2pbob.cc *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Sehraf *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "pqissli2pbob.h"
bool pqissli2pbob::connect_parameter(uint32_t type, const std::string &value)
{
if (type == NET_PARAM_CONNECT_DOMAIN_ADDRESS)
{
RS_STACK_MUTEX(mSslMtx);
// a new line must be appended!
mI2pAddr = value + '\n';
return true;
}
return pqissl::connect_parameter(type, value);
}
int pqissli2pbob::Basic_Connection_Complete()
{
int ret;
if ((ret = pqissl::Basic_Connection_Complete()) != 1)
{
// basic connection not complete.
return ret;
}
// send addr. (new line is already appended)
ret = send(sockfd, mI2pAddr.c_str(), mI2pAddr.length(), 0);
if (ret != (int)mI2pAddr.length())
return -1;
return 1;
}

52
libretroshare/src/pqi/pqissli2pbob.h
View File

@ -1,52 +0,0 @@
/*******************************************************************************
* libretroshare/src/pqi: pqissli2pbob.h *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Sehraf *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef PQISSLI2PBOB_H
#define PQISSLI2PBOB_H
#include "pqi/pqissl.h"
/*
* This class is a minimal varied version of pqissl to work with I2P BOB tunnels.
* The only difference is that the [.b32].i2p addresses must be sent first.
*
* Everything else is untouched.
*/
class pqissli2pbob : public pqissl
{
public:
pqissli2pbob(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm)
: pqissl(l, parent, lm) {}
// NetInterface interface
public:
bool connect_parameter(uint32_t type, const std::string &value);
// pqissl interface
protected:
int Basic_Connection_Complete();
private:
std::string mI2pAddr;
};
#endif // PQISSLI2PBOB_H

261
libretroshare/src/pqi/pqissli2psam3.cpp Normal file
View File

@ -0,0 +1,261 @@
#include "pqissli2psam3.h"
#include <libsam3.h>
RS_SET_CONTEXT_DEBUG_LEVEL(2)
static constexpr int pqiDone = 1;
static constexpr int pqiWait = 0;
static constexpr int pqiError = -1;
pqissli2psam3::pqissli2psam3(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm)
: pqissl(l, parent, lm), mState(pqisslSam3State::NONE), mI2pAddrB32(), mI2pAddrLong()
{
RS_DBG4();
mConn = nullptr;
}
bool pqissli2psam3::connect_parameter(uint32_t type, const std::string &value)
{
RS_DBG4();
if (type == NET_PARAM_CONNECT_DOMAIN_ADDRESS)
{
RS_DBG1("got addr:", value);
RS_STACK_MUTEX(mSslMtx);
mI2pAddrB32 = value;
return true;
}
return pqissl::connect_parameter(type, value);
}
int pqissli2psam3::Initiate_Connection()
{
RS_DBG4();
if(waiting != WAITING_DELAY)
{
RS_ERR("Already Attempt in Progress!");
return pqiError;
}
switch (mState) {
case(pqisslSam3State::NONE):
RS_DBG2("NONE");
{
if(mConn) {
// how did we end up here?
RS_ERR("state is NONE but a connection is existing?!");
}
mConn = 0;
// get SAM session
mConn = 0;
samSettings ss;
ss.session = nullptr;
rsAutoProxyMonitor::taskSync(autoProxyType::I2PSAM3, autoProxyTask::getSettings, static_cast<void*>(&ss));
if (!!ss.session) {
RS_DBG3("NONE->DO_LOOKUP");
mState = pqisslSam3State::DO_LOOKUP;
} else {
RS_DBG3("NONE->DO_LOOKUP NOPE", ss.session);
}
}
break;
case(pqisslSam3State::DO_LOOKUP):
RS_DBG1("DO_LOOKUP");
if (!mI2pAddrLong.empty()) {
// skip lookup, it is highly unlikely/impossible for a public key to change (isn't it?)
mState = pqisslSam3State::WAIT_LOOKUP;
break;
}
{
i2p::address *addr = new i2p::address;
addr->clear();
addr->base32 = mI2pAddrB32;
rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::lookupKey, this, static_cast<void*>(addr));
}
mState = pqisslSam3State::WAIT_LOOKUP;
break;
case(pqisslSam3State::DO_CONNECT):
RS_DBG2("DO_CONNECT");
{
auto wrapper = new samEstablishConnectionWrapper();
wrapper->address.clear();
wrapper->address.publicKey = mI2pAddrLong;
wrapper->connection = nullptr;
rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::establishConnection, this, static_cast<void*>(wrapper));
}
mState = pqisslSam3State::WAIT_CONNECT;
break;
case(pqisslSam3State::DONE):
RS_DBG2("DONE");
if (setupSocket())
return pqiDone;
return pqiError;
/* waiting */
case(pqisslSam3State::WAIT_LOOKUP):
RS_DBG3("WAIT_LOOKUP");
break;
case(pqisslSam3State::WAIT_CONNECT):
RS_DBG3("WAIT_CONNECT");
break;
}
return pqiWait;
}
int pqissli2psam3::net_internal_close(int fd)
{
RS_DBG4();
// sanity check
if (mConn && fd != mConn->fd) {
// this should never happen!
RS_ERR("fd != mConn");
// sam3CloseConnection(mConn);
}
// now to the actuall closing
int ret = pqissl::net_internal_close(fd);
rsAutoProxyMonitor::taskAsync(autoProxyType::I2PSAM3, autoProxyTask::closeConnection, this, mConn);
// finally cleanup
mConn = 0;
mState = pqisslSam3State::NONE;
return ret;
}
void pqissli2psam3::taskFinished(taskTicket *&ticket)
{
RS_DBG4();
switch (ticket->task) {
case autoProxyTask::lookupKey:
{
auto addr = static_cast<i2p::address*>(ticket->data);
RS_STACK_MUTEX(mSslMtx);
if (ticket->result == autoProxyStatus::ok) {
mI2pAddrLong = addr->publicKey;
mState = pqisslSam3State::DO_CONNECT;
} else {
waiting = WAITING_FAIL_INTERFACE;
}
delete addr;
ticket->data = nullptr;
addr = nullptr;
}
break;
case autoProxyTask::establishConnection:
{
auto wrapper = static_cast<struct samEstablishConnectionWrapper*>(ticket->data);
RS_STACK_MUTEX(mSslMtx);
if (ticket->result == autoProxyStatus::ok) {
mConn = wrapper->connection;
mState = pqisslSam3State::DONE;
} else {
waiting = WAITING_FAIL_INTERFACE;
}
delete wrapper;
ticket->data = nullptr;
wrapper = nullptr;
}
break;
case autoProxyTask::closeConnection:
// nothing to do here
break;
default:
RS_WARN("unkown task", ticket->task);
}
// clean up!
delete ticket;
ticket = nullptr;
}
bool pqissli2psam3::setupSocket()
{
/*
* This function contains the generis part from pqissl::Initiate_Connection()
*/
int err;
int osock = mConn->fd;
err = unix_fcntl_nonblock(osock);
if (err < 0)
{
RS_ERR("Cannot make socket NON-Blocking:", err);
waiting = WAITING_FAIL_INTERFACE;
net_internal_close(osock);
return false;
}
#ifdef WINDOWS_SYS
/* Set TCP buffer size for Windows systems */
int sockbufsize = 0;
int size = sizeof(int);
err = getsockopt(osock, SOL_SOCKET, SO_RCVBUF, (char *)&sockbufsize, &size);
#ifdef PQISSL_DEBUG
if (err == 0) {
std::cerr << "pqissl::Initiate_Connection: Current TCP receive buffer size " << sockbufsize << std::endl;
} else {
std::cerr << "pqissl::Initiate_Connection: Error getting TCP receive buffer size. Error " << err << std::endl;
}
#endif
sockbufsize = 0;
err = getsockopt(osock, SOL_SOCKET, SO_SNDBUF, (char *)&sockbufsize, &size);
#ifdef PQISSL_DEBUG
if (err == 0) {
std::cerr << "pqissl::Initiate_Connection: Current TCP send buffer size " << sockbufsize << std::endl;
} else {
std::cerr << "pqissl::Initiate_Connection: Error getting TCP send buffer size. Error " << err << std::endl;
}
#endif
sockbufsize = WINDOWS_TCP_BUFFER_SIZE;
err = setsockopt(osock, SOL_SOCKET, SO_RCVBUF, (char *)&sockbufsize, sizeof(sockbufsize));
#ifdef PQISSL_DEBUG
if (err == 0) {
std::cerr << "pqissl::Initiate_Connection: TCP receive buffer size set to " << sockbufsize << std::endl;
} else {
std::cerr << "pqissl::Initiate_Connection: Error setting TCP receive buffer size. Error " << err << std::endl;
}
#endif
err = setsockopt(osock, SOL_SOCKET, SO_SNDBUF, (char *)&sockbufsize, sizeof(sockbufsize));
#ifdef PQISSL_DEBUG
if (err == 0) {
std::cerr << "pqissl::Initiate_Connection: TCP send buffer size set to " << sockbufsize << std::endl;
} else {
std::cerr << "pqissl::Initiate_Connection: Error setting TCP send buffer size. Error " << err << std::endl;
}
#endif
#endif // WINDOWS_SYS
mTimeoutTS = time(NULL) + mConnectTimeout;
//std::cerr << "Setting Connect Timeout " << mConnectTimeout << " Seconds into Future " << std::endl;
waiting = WAITING_SOCK_CONNECT;
sockfd = osock;
return true;
}

47
libretroshare/src/pqi/pqissli2psam3.h Normal file
View File

@ -0,0 +1,47 @@
#ifndef PQISSLI2PSAM3_H
#define PQISSLI2PSAM3_H
#include "pqi/pqissl.h"
#include "services/autoproxy/rsautoproxymonitor.h"
#include "services/autoproxy/p3i2psam3.h"
// Use a state machine as the whole pqi code is designed around them and some operation (like lookup) might be blocking
enum class pqisslSam3State : uint8_t {
NONE = 0,
DO_LOOKUP,
WAIT_LOOKUP,
DO_CONNECT,
WAIT_CONNECT,
DONE
};
class pqissli2psam3 : public pqissl, public autoProxyCallback
{
public:
pqissli2psam3(pqissllistener *l, PQInterface *parent, p3LinkMgr *lm);
// NetInterface interface
public:
bool connect_parameter(uint32_t type, const std::string &value);
// pqissl interface
protected:
int Initiate_Connection();
int net_internal_close(int fd);
// autoProxyCallback interface
public:
void taskFinished(taskTicket *&ticket);
private:
bool setupSocket();
private:
pqisslSam3State mState;
std::string mI2pAddrB32;
std::string mI2pAddrLong;
Sam3Connection *mConn;
};
#endif // PQISSLI2PSAM3_H

4
libretroshare/src/pqi/pqissllistener.h
View File

@ -186,8 +186,8 @@ public:
virtual int finaliseConnection(int fd, SSL *ssl, const RsPeerId& peerId,
const sockaddr_storage &raddr);
RS_SET_CONTEXT_DEBUG_LEVEL(2)
private:
std::map<RsPeerId, pqissl*> listenaddr;
RS_SET_CONTEXT_DEBUG_LEVEL(2)
};

22
libretroshare/src/pqi/pqisslpersongrp.cc
View File

@ -46,7 +46,7 @@ static struct RsLog::logInfo pqipersongrpzoneInfo = {RsLog::Default, "pqipersong
#endif
#include "pqi/pqisslproxy.h"
#include "pqi/pqissli2pbob.h"
#include "pqi/pqissli2psam3.h"
pqilistener * pqisslpersongrp::locked_createListener(const struct sockaddr_storage &laddr)
{
@ -74,24 +74,26 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
std::cerr << std::endl;
#endif
// Use pqicI2PBOB for I2P
pqiconnect *pqicSOCKSProxy, *pqicI2PBOB;
// Use pqicI2P for I2P
pqiconnect *pqicSOCKSProxy, *pqicI2P;
{
pqisslproxy *pqis = new pqisslproxy((pqissllistener *) listener, pqip, mLinkMgr);
RsSerialiser *rss = new RsSerialiser();
rss->addSerialType(new RsRawSerialiser());
pqicSOCKSProxy = new pqiconnect(pqip, rss, pqis);
}
if (rsAutoProxyMonitor::instance()->isEnabled(autoProxyType::I2PBOB))
#ifdef RS_USE_I2P_SAM3
if (rsAutoProxyMonitor::instance()->isEnabled(autoProxyType::I2PSAM3))
{
pqissli2pbob *pqis = new pqissli2pbob((pqissllistener *) listener, pqip, mLinkMgr);
pqissli2psam3 *pqis = new pqissli2psam3((pqissllistener *) listener, pqip, mLinkMgr);
RsSerialiser *rss = new RsSerialiser();
rss->addSerialType(new RsRawSerialiser());
pqicI2PBOB = new pqiconnect(pqip, rss, pqis);
} else {
pqicI2PBOB = pqicSOCKSProxy;
pqicI2P = new pqiconnect(pqip, rss, pqis);
}
else
#endif // def RS_USE_I2P_SAM3
pqicI2P = pqicSOCKSProxy;
/* first select type based on peer */
@ -101,7 +103,7 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_TOR_TCP, pqicSOCKSProxy);
break;
case RS_HIDDEN_TYPE_I2P:
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2PBOB);
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2P);
break;
default:
/* peer is not a hidden one but we are */
@ -109,7 +111,7 @@ pqiperson * pqisslpersongrp::locked_createPerson(const RsPeerId& id, pqilistener
uint32_t typeOwn = mPeerMgr->getHiddenType(AuthSSL::getAuthSSL()->OwnId());
switch (typeOwn) {
case RS_HIDDEN_TYPE_I2P:
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2PBOB);
pqip -> addChildInterface(PQI_CONNECT_HIDDEN_I2P_TCP, pqicI2P);
break;
default:
/* this case shouldn't happen! */

11
libretroshare/src/retroshare/rsevents.h
View File

@ -91,8 +91,8 @@ enum class RsEventType : uint32_t
/// @see RsGxsPostedEvent
GXS_IDENTITY = 12,
/// @see RsFiles
SHARED_DIRECTORIES = 13,
/// @see RsFiles @deprecated
SHARED_DIRECTORIES = 13,
/// @see RsFiles
FILE_TRANSFER = 14,
@ -100,8 +100,11 @@ enum class RsEventType : uint32_t
/// @see RsMsgs
CHAT_MESSAGE = 15,
/// @see rspeers.h
NETWORK = 16,
/// @see rspeers.h
NETWORK = 16,
/** Emitted to update library clients about file hashing being completed */
FILE_HASHING_COMPLETED = 20,
__MAX /// Used internally, keep last
};

28
libretroshare/src/retroshare/rsfiles.h
View File

@ -194,7 +194,8 @@ enum class RsFileTransferEventCode: uint8_t {
COMPLETED_FILES_REMOVED = 0x02, //
};
struct RsSharedDirectoriesEvent: RsEvent
struct RS_DEPRECATED_FOR("Packing arbitrary data into an std::string is bad idea")
RsSharedDirectoriesEvent: RsEvent
{
RsSharedDirectoriesEvent() : RsEvent(RsEventType::SHARED_DIRECTORIES), mEventCode(RsSharedDirectoriesEventCode::UNKNOWN) {}
~RsSharedDirectoriesEvent() override = default;
@ -212,6 +213,31 @@ struct RsSharedDirectoriesEvent: RsEvent
std::string mMessage;
};
struct RsFileHashingCompletedEvent: RsEvent
{
RsFileHashingCompletedEvent():
RsEvent(RsEventType::FILE_HASHING_COMPLETED), mHashingSpeed(0) {}
///* @see RsEvent @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RsEvent::serial_process(j, ctx);
RS_SERIAL_PROCESS(mFilePath);
RS_SERIAL_PROCESS(mFileHash);
RS_SERIAL_PROCESS(mHashingSpeed);
}
/// Complete path of the file being hashed
std::string mFilePath;
/// File hash, null if error occurred
RsFileHash mFileHash;
/// Hashing speed in MB/s
double mHashingSpeed;
};
struct RsFileTransferEvent: RsEvent
{
RsFileTransferEvent() : RsEvent(RsEventType::FILE_TRANSFER), mFileTransferEventCode(RsFileTransferEventCode::UNKNOWN) {}

5
libretroshare/src/retroshare/rsgxscircles.h
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -46,7 +47,7 @@ extern RsGxsCircles* rsGxsCircles;
enum class RsGxsCircleType : uint32_t // 32 bit overkill, just for retrocompat
{
UNKNOWN = 0, /// Used to detect uninizialized values.
PUBLIC = 1, /// Public distribution, based on GxsIds
PUBLIC = 1, /// Public distribution
EXTERNAL = 2, /// Restricted to an external circle, based on GxsIds
NODES_GROUP = 3, /// Restricted to a group of friend nodes, the administrator of the circle behave as a hub for them

86
libretroshare/src/retroshare/rsgxsforums.h
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -115,9 +115,13 @@ enum class RsForumEventCode: uint8_t
READ_STATUS_CHANGED = 0x06, /// msg was read or marked unread
STATISTICS_CHANGED = 0x07, /// suppliers and how many messages they have changed
MODERATOR_LIST_CHANGED = 0x08, /// forum moderation list has changed.
SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed
PINNED_POSTS_CHANGED = 0x0b, /// some posts where pinned or un-pinned
DELETED_FORUM = 0x0c, /// forum was deleted by cleaning
SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed
PINNED_POSTS_CHANGED = 0x0b, /// some posts where pinned or un-pinned
DELETED_FORUM = 0x0c, /// forum was deleted by cleaning
DELETED_POST = 0x0d, /// Post deleted (usually by cleaning)
/// Distant search result received
DISTANT_SEARCH_RESULT = 0x0e
};
struct RsGxsForumEvent: RsEvent
@ -129,8 +133,8 @@ struct RsGxsForumEvent: RsEvent
RsForumEventCode mForumEventCode;
RsGxsGroupId mForumGroupId;
RsGxsMessageId mForumMsgId;
std::list<RsGxsId> mModeratorsAdded;
std::list<RsGxsId> mModeratorsRemoved;
std::list<RsGxsId> mModeratorsAdded;
std::list<RsGxsId> mModeratorsRemoved;
///* @see RsEvent @see RsSerializable
void serial_process(
@ -141,7 +145,6 @@ struct RsGxsForumEvent: RsEvent
RS_SERIAL_PROCESS(mForumEventCode);
RS_SERIAL_PROCESS(mForumGroupId);
RS_SERIAL_PROCESS(mForumMsgId);
RS_SERIAL_PROCESS(mForumMsgId);
RS_SERIAL_PROCESS(mModeratorsAdded);
RS_SERIAL_PROCESS(mModeratorsRemoved);
}
@ -149,6 +152,29 @@ struct RsGxsForumEvent: RsEvent
~RsGxsForumEvent() override;
};
/** This event is fired once distant search results are received */
struct RsGxsForumsDistantSearchEvent: RsEvent
{
RsGxsForumsDistantSearchEvent():
RsEvent(RsEventType::GXS_FORUMS),
mForumEventCode(RsForumEventCode::DISTANT_SEARCH_RESULT) {}
RsForumEventCode mForumEventCode;
TurtleRequestId mSearchId;
std::vector<RsGxsSearchResult> mSearchResults;
///* @see RsEvent @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RsEvent::serial_process(j, ctx);
RS_SERIAL_PROCESS(mForumEventCode);
RS_SERIAL_PROCESS(mSearchId);
RS_SERIAL_PROCESS(mSearchResults);
}
};
class RsGxsForums: public RsGxsIfaceHelper
{
public:
@ -385,6 +411,50 @@ public:
const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
bool keepForever ) = 0;
/**
* @brief Get forum content summaries
* @jsonapi{development}
* @param[in] forumId id of the forum of which the content is requested
* @param[in] contentIds ids of requested contents, if empty summaries of
* all messages are reqeusted
* @param[out] summaries storage for summaries
* @return success or error details if something failed
*/
virtual std::error_condition getContentSummaries(
const RsGxsGroupId& forumId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) = 0;
/**
* @brief Search the whole reachable network for matching forums and
* posts
* @jsonapi{development}
* An @see RsGxsForumsDistantSearchEvent is emitted when matching results
* arrives from the network
* @param[in] matchString string to search into the forum and posts
* @param[out] searchId storage for search id, useful to track search events
* and retrieve search results
* @return success or error details
*/
virtual std::error_condition distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId ) = 0;
/**
* @brief Search the local index for matching forums and posts
* @jsonapi{development}
* @param[in] matchString string to search into the index
* @param[out] searchResults storage for searchr esults
* @return success or error details
*/
virtual std::error_condition localSearch(
const std::string& matchString,
std::vector<RsGxsSearchResult>& searchResults ) = 0;
////////////////////////////////////////////////////////////////////////////
/* Following functions are deprecated and should not be considered a stable
* to use API */
/**
* @brief Create forum. Blocking API.
* @jsonapi{development}

62
libretroshare/src/retroshare/rsgxsiface.h
View File

@ -4,7 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012 Christopher Evi-Parker *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -31,13 +32,63 @@
#include "rsitems/rsserviceids.h"
#include "retroshare/rsevents.h"
/*!
* This structure is used to transport GXS search results.
* It contains the group information as well as a context string to tell where
* the information was found.
* Keep it small as to make search responses as light as possible.
* It differs from RsGxsGroupSearchResults because it supports also results from
* message matches not just groups.
*/
struct RsGxsSearchResult : RsSerializable
{
RsGxsSearchResult(): mPublishTs(0) {}
/** Id of the group which match*/
RsGxsGroupId mGroupId;
/** Title of the group which match */
std::string mGroupName;
/** Optional message id if the search match is against a message */
RsGxsMessageId mMsgId;
/** Optional message title if the search match is against a message */
std::string mMsgName;
/** Author id of the element which matched (group or message) */
RsGxsId mAuthorId;
/** Publish timestamp of the element which matched (group or message) */
rstime_t mPublishTs;
/** A snippet of content around the exact match */
std::string mSearchContext;
/// @see RsSerializable::serial_process
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
{
RS_SERIAL_PROCESS(mGroupId);
RS_SERIAL_PROCESS(mGroupName);
RS_SERIAL_PROCESS(mMsgId);
RS_SERIAL_PROCESS(mMsgName);
RS_SERIAL_PROCESS(mAuthorId);
RS_SERIAL_PROCESS(mPublishTs);
RS_SERIAL_PROCESS(mSearchContext);
}
virtual ~RsGxsSearchResult() = default;
};
/*!
* This structure is used to transport group summary information when a GXS
* service is searched. It contains the group information as well as a context
* string to tell where the information was found. It is more compact than a
* GroupMeta object, so as to make search responses as light as possible.
*/
struct RsGxsGroupSummary : RsSerializable
struct RS_DEPRECATED_FOR(RsGxsSearchResult)
RsGxsGroupSummary : RsSerializable
{
RsGxsGroupSummary() :
mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0),
@ -78,8 +129,12 @@ struct RsGxsGroupSummary : RsSerializable
* strings to tell where the information was found. It is more compact than a
* GroupMeta object, so as to make search responses as light as possible.
*/
struct RsGxsGroupSearchResults : RsSerializable
struct RS_DEPRECATED_FOR(RsGxsSearchResult)
RsGxsGroupSearchResults : RsSerializable
{
/* TODO: This seems exactly the same as RsGxsGroupSummary + mSearchContexts
* do we really need both? */
RsGxsGroupSearchResults()
: mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0)
{}
@ -113,6 +168,7 @@ struct RsGxsGroupSearchResults : RsSerializable
virtual ~RsGxsGroupSearchResults() = default;
};
/*!
* Stores ids of changed gxs groups and messages.
* It is used to notify about GXS changes.

2
libretroshare/src/retroshare/rsgxsifacetypes.h
View File

@ -63,7 +63,7 @@ struct RsGroupMetaData : RsSerializable
mCircleType(0x0001), mAuthenFlags(0), mSubscribeFlags(0), mPop(0),
mVisibleMsgCount(0), mLastPost(0), mGroupStatus(0) {}
virtual ~RsGroupMetaData() {}
virtual ~RsGroupMetaData() = default;
void operator =(const RsGxsGrpMetaData& rGxsMeta);
RsGroupMetaData(const RsGxsGrpMetaData& rGxsMeta) { operator=(rGxsMeta); }

2
libretroshare/src/retroshare/rsinit.h
View File

@ -195,7 +195,7 @@ public:
/*
* Setup Hidden Location;
*/
static void SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useBob);
static void SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useI2p);
static bool LoadPassword(const std::string& passwd) ;

1
libretroshare/src/retroshare/rspeers.h
View File

@ -788,6 +788,7 @@ public:
virtual bool setProxyServer(const uint32_t type, const std::string &addr, const uint16_t port) = 0;
virtual void getIPServersList(std::list<std::string>& ip_servers) = 0;
virtual void getCurrentExtIPList(std::list<std::string>& ip_list) = 0;
virtual void allowServerIPDetermination(bool) = 0;
virtual bool resetOwnExternalAddressList() = 0;
virtual bool getAllowServerIPDetermination() = 0 ;

57
libretroshare/src/retroshare/rstypes.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2004-2006 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2004-2006 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -135,17 +137,10 @@ struct PeerBandwidthLimits : RsSerializable
}
};
//class SearchRequest // unused stuff.
//{
// public:
// int searchId;
// RsCertId toId; /* all zeros for everyone! */
// std::list<Condition> tests;
//};
/********************** For FileCache Interface *****************/
/* G10h4ck: Having this kind of stuff on public headers is pretty dangerous for
* name space pollution, a C++11 enum class should be used instad ASAP */
#define DIR_TYPE_UNKNOWN 0x00
#define DIR_TYPE_ROOT 0x01
#define DIR_TYPE_PERSON 0x02
@ -258,7 +253,10 @@ struct DirStub : RsSerializable
uint8_t type;
std::string name;
void *ref;
/* G10h4ck do we still need to keep this as void* instead of uint64_t for
* retroshare-gui sake? */
void* ref;
/// @see RsSerializable
void serial_process(RsGenericSerializer::SerializeJob j,
@ -267,29 +265,12 @@ struct DirStub : RsSerializable
RS_SERIAL_PROCESS(type);
RS_SERIAL_PROCESS(name);
#if defined(__GNUC__) && !defined(__clang__)
# pragma GCC diagnostic ignored "-Wstrict-aliasing"
#endif // defined(__GNUC__) && !defined(__clang__)
// (Cyril) We have to do this because on some systems (MacOS) uintptr_t is unsigned long which is not well defined. It is always
// preferable to force type serialization to the correct size rather than letting the compiler choose for us.
// /!\ This structure cannot be sent over the network. The serialization would be inconsistent.
if(sizeof(ref) == 4)
{
std::uint32_t& handle(reinterpret_cast<std::uint32_t&>(ref));
RS_SERIAL_PROCESS(handle);
}
else if(sizeof(ref) == 8)
{
std::uint64_t& handle(reinterpret_cast<std::uint64_t&>(ref));
RS_SERIAL_PROCESS(handle);
}
else
std::cerr << __PRETTY_FUNCTION__ << ": cannot serialize raw pointer of size " << sizeof(ref) << std::endl;
#if defined(__GNUC__) && !defined(__clang__)
# pragma GCC diagnostic pop
#endif // defined(__GNUC__) && !defined(__clang__)
/* Enforce serialization as uint64_t because void* changes size (usually
* 4 bytes on 32bit arch and 8 bytes on 64bit archs) depending on
* architectures and make JSON API behave inconsistenly. */
uint64_t handle = reinterpret_cast<uint64_t>(ref);
RS_SERIAL_PROCESS(handle);
ref = reinterpret_cast<void*>(handle);
}
};
@ -330,9 +311,9 @@ struct DirDetails : RsSerializable
void serial_process(RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx) override
{
/* Enforce serialization as uint64_t because void* changes size
* depending (usually 4 bytes on 32bit arch and 8 bytes on 64bit archs)
*/
/* Enforce serialization as uint64_t because void* changes size (usually
* 4 bytes on 32bit arch and 8 bytes on 64bit archs) depending on
* architectures and make JSON API behave inconsistenly. */
uint64_t handle = reinterpret_cast<uint64_t>(ref);
RS_SERIAL_PROCESS(handle);
ref = reinterpret_cast<void*>(handle);
@ -347,7 +328,7 @@ struct DirDetails : RsSerializable
RS_SERIAL_PROCESS(name);
RS_SERIAL_PROCESS(hash);
RS_SERIAL_PROCESS(path);
RS_SERIAL_PROCESS(size);
RS_SERIAL_PROCESS(size);
RS_SERIAL_PROCESS(mtime);
RS_SERIAL_PROCESS(flags);
RS_SERIAL_PROCESS(max_mtime);

62
libretroshare/src/rsitems/rsgxsforumitems.h
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2012 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +21,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RS_GXS_FORUM_ITEMS_H
#define RS_GXS_FORUM_ITEMS_H
#pragma once
#include <map>
@ -31,7 +32,18 @@
#include "retroshare/rsgxsforums.h"
enum class RsGxsForumsItems : uint8_t
{
GROUP_ITEM = 0x02,
MESSAGE_ITEM = 0x03,
SEARCH_REQUEST = 0x04,
SEARCH_REPLY = 0x05,
};
RS_DEPRECATED_FOR(RsGxsForumsItems)
const uint8_t RS_PKT_SUBTYPE_GXSFORUM_GROUP_ITEM = 0x02;
RS_DEPRECATED_FOR(RsGxsForumsItems)
const uint8_t RS_PKT_SUBTYPE_GXSFORUM_MESSAGE_ITEM = 0x03;
class RsGxsForumGroupItem : public RsGxsGrpItem
@ -61,6 +73,48 @@ public:
RsGxsForumMsg mMsg;
};
struct RsGxsForumsSearchRequest : RsSerializable
{
RsGxsForumsSearchRequest() : mType(RsGxsForumsItems::SEARCH_REQUEST) {}
/// Just for easier back and forward compatibility
RsGxsForumsItems mType;
/// Store search match string
std::string mQuery;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mQuery);
}
~RsGxsForumsSearchRequest() override = default;
};
struct RsGxsForumsSearchReply : RsSerializable
{
RsGxsForumsSearchReply() : mType(RsGxsForumsItems::SEARCH_REPLY) {}
/// Just for easier back and forward compatibility
RsGxsForumsItems mType;
/// Results storage
std::vector<RsGxsSearchResult> mResults;
/// @see RsSerializable
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mType);
RS_SERIAL_PROCESS(mResults);
}
~RsGxsForumsSearchReply() override = default;
};
class RsGxsForumSerialiser : public RsServiceSerializer
{
public:
@ -69,5 +123,3 @@ public:
virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ;
};
#endif /* RS_GXS_FORUM_ITEMS_H */

5
libretroshare/src/rsitems/rsitem.h
View File

@ -86,9 +86,8 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable
virtual void serial_process(RsGenericSerializer::SerializeJob,
RsGenericSerializer::SerializeContext&)// = 0;
{
std::cerr << "(EE) RsItem::serial_process(...) called by an item using"
<< "new serialization classes, but not derived! Class is "
<< typeid(*this).name() << std::endl;
RS_ERR( "called by an item using new serialization system without "
"overriding Class is: ", typeid(*this).name() );
print_stacktrace();
}

14
libretroshare/src/rsitems/rsserviceids.h
View File

@ -28,7 +28,9 @@
enum class RsServiceType : uint16_t
{
NONE = 0, /// To detect non-initialized reads
/// To detect non-initialized items
NONE = 0,
GOSSIP_DISCOVERY = 0x0011,
CHAT = 0x0012,
MSG = 0x0013,
@ -46,7 +48,10 @@ enum class RsServiceType : uint16_t
GWEMAIL_MAIL = 0x0025,
SERVICE_CONTROL = 0x0026,
DISTANT_CHAT = 0x0027,
/// For GXS identity authenticated tunnels, do not confuse with @GXS_DISTANT
GXS_TUNNEL = 0x0028,
BANLIST = 0x0101,
STATUS = 0x0102,
FRIEND_SERVER = 0x0103,
@ -59,6 +64,7 @@ enum class RsServiceType : uint16_t
POSTED = 0x0216,
CHANNELS = 0x0217,
GXSCIRCLE = 0x0218,
/// not gxs, but used with identities.
REPUTATION = 0x0219,
GXS_RECOGN = 0x0220,
@ -69,13 +75,13 @@ enum class RsServiceType : uint16_t
CHANNELS_CONFIG = 0x0317,
RTT = 0x1011, /// Round Trip Time
/***************** IDS ALLOCATED FOR PLUGINS ******************/
// 2000+
PLUGIN_ARADO_ID = 0x2001,
PLUGIN_QCHESS_ID = 0x2002,
PLUGIN_FEEDREADER = 0x2003,
/// GXS distant sync and search do not confuse with @see GXS_TUNNEL
GXS_DISTANT = 0x2233,
/// Reserved for packet slicing probes.
PACKET_SLICING_PROBE = 0xAABB,

6
libretroshare/src/rsserver/p3face.h
View File

@ -43,7 +43,7 @@
class p3heartbeat;
class p3discovery2;
class p3I2pBob;
class p3I2pSam3;
/* GXS Classes - just declare the classes.
so we don't have to totally recompile to switch */
@ -161,8 +161,8 @@ public:
p3ChatService *chatSrv;
p3StatusService *mStatusSrv;
p3GxsTunnelService *mGxsTunnels;
#ifdef RS_USE_I2P_BOB
p3I2pBob *mI2pBob;
#ifdef RS_USE_I2P_SAM3
p3I2pSam3 *mI2pSam3;
#endif
// This list contains all threaded services. It will be used to shut them down properly.

59
libretroshare/src/rsserver/p3peers.cc
View File

@ -374,7 +374,7 @@ bool p3Peers::getPeerDetails(const RsPeerId& id, RsPeerDetails &d)
sockaddr_storage_ipv6_to_ipv4(it->mAddr);
std::string toto;
toto += sockaddr_storage_tostring(it->mAddr);
rs_sprintf_append(toto, " %ld sec", time(NULL) - it->mSeenTime);
rs_sprintf_append(toto, " %ld sec loc", time(NULL) - it->mSeenTime);
d.ipAddressList.push_back(toto);
}
for(it = ps.ipAddrs.mExt.mAddrs.begin(); it != ps.ipAddrs.mExt.mAddrs.end(); ++it)
@ -382,7 +382,7 @@ bool p3Peers::getPeerDetails(const RsPeerId& id, RsPeerDetails &d)
sockaddr_storage_ipv6_to_ipv4(it->mAddr);
std::string toto;
toto += sockaddr_storage_tostring(it->mAddr);
rs_sprintf_append(toto, " %ld sec", time(NULL) - it->mSeenTime);
rs_sprintf_append(toto, " %ld sec ext", time(NULL) - it->mSeenTime);
d.ipAddressList.push_back(toto);
}
}
@ -860,9 +860,15 @@ void p3Peers::getIPServersList(std::list<std::string>& ip_servers)
{
mNetMgr->getIPServersList(ip_servers) ;
}
void p3Peers::getCurrentExtIPList(std::list<std::string>& ip_list)
{
mNetMgr->getCurrentExtIPList(ip_list) ;
}
bool p3Peers::resetOwnExternalAddressList()
{
return mPeerMgr->resetOwnExternalAddressList();
//TODO Phenom 2021-10-30: Need to call something like mNetMgr->netReset();
// to update this addresslist.
return mPeerMgr->resetOwnExternalAddressList();
}
void p3Peers::allowServerIPDetermination(bool b)
{
@ -1272,7 +1278,11 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
}
#else
sockaddr_storage tLocal;
if(sockaddr_storage_inet_pton(tLocal, tDetails.localAddr) && sockaddr_storage_isValidNet(tLocal) && sockaddr_storage_ipv6_to_ipv4(tLocal) && tDetails.localPort )
bool validLoc = sockaddr_storage_inet_pton(tLocal, tDetails.localAddr)
&& sockaddr_storage_isValidNet(tLocal)
&& tDetails.localPort;
bool isLocIpv4 = sockaddr_storage_ipv6_to_ipv4(tLocal);
if(validLoc && isLocIpv4)
{
uint32_t t4Addr = reinterpret_cast<sockaddr_in&>(tLocal).sin_addr.s_addr;
@ -1290,7 +1300,11 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
}
sockaddr_storage tExt;
if(sockaddr_storage_inet_pton(tExt, tDetails.extAddr) && sockaddr_storage_isValidNet(tExt) && sockaddr_storage_ipv6_to_ipv4(tExt) && tDetails.extPort )
bool validExt = sockaddr_storage_inet_pton(tExt, tDetails.extAddr)
&& sockaddr_storage_isValidNet(tExt)
&& tDetails.extPort;
bool isExtIpv4 = sockaddr_storage_ipv6_to_ipv4(tExt);
if(validExt && isExtIpv4)
{
uint32_t t4Addr = reinterpret_cast<sockaddr_in&>(tExt).sin_addr.s_addr;
@ -1306,6 +1320,17 @@ bool p3Peers::getShortInvite(std::string& invite, const RsPeerId& _sslId, Retros
offset += 4+2;
}
else if(validExt && !isExtIpv4)
{
// External address is IPv6, save it on LOCATOR
sockaddr_storage_setport(tExt,tDetails.extPort);
std::string tLocator = sockaddr_storage_tostring(tExt);
addPacketHeader(RsShortInviteFieldType::LOCATOR, tLocator.size(),buf,offset,buf_size);
memcpy(&buf[offset],tLocator.c_str(),tLocator.size());
offset += tLocator.size();
}
#endif
}
@ -1457,13 +1482,31 @@ std::string p3Peers::GetRetroshareInvite( const RsPeerId& sslId, RetroshareInvit
if (getPeerDetails(ssl_id, detail))
{
if(!(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY) || detail.isHiddenNode)
detail.ipAddressList.clear();
if( !(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY)
|| detail.isHiddenNode)
detail.ipAddressList.clear();
//Check if external address is IPv6, then move it to ipAddressList as RsCertificate only allow 4 numbers.
sockaddr_storage tExt;
bool validExt = sockaddr_storage_inet_pton(tExt, detail.extAddr)
&& sockaddr_storage_isValidNet(tExt)
&& detail.extPort;
bool isExtIpv4 = sockaddr_storage_ipv6_to_ipv4(tExt);
if( !(invite_flags & RetroshareInviteFlags::FULL_IP_HISTORY)
&& !detail.isHiddenNode
&& validExt && !isExtIpv4)
{
sockaddr_storage_setport(tExt,detail.extPort);
detail.ipAddressList.push_front(sockaddr_storage_tostring(tExt) + " "); // Space needed to later parse.
detail.extAddr = ""; //Clear it to not trigg error.
detail.extPort = 0;
}
unsigned char *mem_block = nullptr;
size_t mem_block_size = 0;
if(!AuthGPG::getAuthGPG()->exportPublicKey( RsPgpId(detail.gpg_id), mem_block, mem_block_size, false, !!(invite_flags & RetroshareInviteFlags::PGP_SIGNATURES) ))
if(!AuthGPG::getAuthGPG()->exportPublicKey( RsPgpId(detail.gpg_id), mem_block, mem_block_size, false, !!(invite_flags & RetroshareInviteFlags::PGP_SIGNATURES) ))
{
std::cerr << "Cannot output certificate for id \"" << detail.gpg_id
<< "\". Sorry." << std::endl;

1
libretroshare/src/rsserver/p3peers.h
View File

@ -132,6 +132,7 @@ public:
virtual bool isProxyAddress(const uint32_t type, const sockaddr_storage &addr);
virtual void getIPServersList(std::list<std::string>& ip_servers) override;
virtual void getCurrentExtIPList(std::list<std::string>& ip_list) override;
virtual void allowServerIPDetermination(bool) override;
virtual bool getAllowServerIPDetermination() override;
virtual bool resetOwnExternalAddressList() override;

86
libretroshare/src/rsserver/rsinit.cc
View File

@ -2,7 +2,8 @@
* libretroshare/src/retroshare: rsinit.cc *
* *
* Copyright (C) 2004-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2016-2019 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2016-2021 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -171,7 +172,7 @@ struct RsInitConfig
std::string hiddenNodeAddress;
uint16_t hiddenNodePort;
bool hiddenNodeI2PBOB;
bool hiddenNodeI2P;
/* Logging */
bool haveLogFile;
@ -665,13 +666,13 @@ void RsInit::setAutoLogin(bool autoLogin){
}
/* Setup Hidden Location; */
void RsInit::SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useBob)
void RsInit::SetHiddenLocation(const std::string& hiddenaddress, uint16_t port, bool useI2p)
{
/* parse the bugger (todo) */
rsInitConfig->hiddenNodeSet = true;
rsInitConfig->hiddenNodeAddress = hiddenaddress;
rsInitConfig->hiddenNodePort = port;
rsInitConfig->hiddenNodeI2PBOB = useBob;
rsInitConfig->hiddenNodeI2P = useI2p;
}
@ -718,7 +719,7 @@ RsGRouter *rsGRouter = NULL ;
# include "rs_upnp/upnphandler_miniupnp.h"
#endif // def RS_USE_LIBUPNP
#include "services/autoproxy/p3i2pbob.h"
#include "services/autoproxy/p3i2psam3.h"
#include "services/autoproxy/rsautoproxymonitor.h"
#include "services/p3gxsreputation.h"
@ -924,9 +925,9 @@ int RsServer::StartupRetroShare()
mNetMgr->setManagers(mPeerMgr, mLinkMgr);
rsAutoProxyMonitor *autoProxy = rsAutoProxyMonitor::instance();
#ifdef RS_USE_I2P_BOB
mI2pBob = new p3I2pBob(mPeerMgr);
autoProxy->addProxy(autoProxyType::I2PBOB, mI2pBob);
#ifdef RS_USE_I2P_SAM3
mI2pSam3 = new p3I2pSam3(mPeerMgr);
autoProxy->addProxy(autoProxyType::I2PSAM3, mI2pSam3);
#endif
//load all the SSL certs as friends
@ -1340,22 +1341,26 @@ int RsServer::StartupRetroShare()
mWiki->setNetworkExchangeService(wiki_ns) ;
#endif
/**** Forum GXS service ****/
/************************* Forum GXS service ******************************/
RsGeneralDataService* gxsforums_ds = new RsDataService(currGxsDir + "/", "gxsforums_db",
RS_SERVICE_GXS_TYPE_FORUMS, NULL, rsInitConfig->gxs_passwd);
RsGeneralDataService* gxsforums_ds = new RsDataService(
currGxsDir + "/", "gxsforums_db", RS_SERVICE_GXS_TYPE_FORUMS,
nullptr, rsInitConfig->gxs_passwd );
p3GxsForums* mGxsForums = new p3GxsForums(
gxsforums_ds, nullptr, mGxsIdService );
p3GxsForums *mGxsForums = new p3GxsForums(gxsforums_ds, NULL, mGxsIdService);
RsGxsNetTunnelService* gxsForumsTunnelService = nullptr;
#ifdef RS_DEEP_FORUMS_INDEX
gxsForumsTunnelService = mGxsNetTunnel;
#endif
// create GXS photo service
RsGxsNetService* gxsforums_ns = new RsGxsNetService(
RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr,
mGxsForums, mGxsForums->getServiceInfo(),
mReputations, mGxsCircles,mGxsIdService,
pgpAuxUtils);//,mGxsNetTunnel,true,true,true);
RsGxsNetService* gxsforums_ns = new RsGxsNetService(
RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr, mGxsForums,
mGxsForums->getServiceInfo(), mReputations, mGxsCircles,
mGxsIdService, pgpAuxUtils, gxsForumsTunnelService );
mGxsForums->setNetworkExchangeService(gxsforums_ns);
mGxsForums->setNetworkExchangeService(gxsforums_ns) ;
/**** Channel GXS service ****/
@ -1602,7 +1607,10 @@ int RsServer::StartupRetroShare()
/**************************************************************************/
// Turtle search for GXS services
mGxsNetTunnel->registerSearchableService(gxschannels_ns) ;
mGxsNetTunnel->registerSearchableService(gxschannels_ns);
#ifdef RS_DEEP_FORUMS_INDEX
mGxsNetTunnel->registerSearchableService(gxsforums_ns);
#endif
/**************************************************************************/
@ -1659,8 +1667,9 @@ int RsServer::StartupRetroShare()
mConfigMgr->addConfiguration("wire.cfg", wire_ns);
#endif
#endif //RS_ENABLE_GXS
#ifdef RS_USE_I2P_BOB
mConfigMgr->addConfiguration("I2PBOB.cfg", mI2pBob);
#ifdef RS_USE_I2P_SAM3
// to make migration easiert, SAM will use BOBs configuration, as they are compatible / the same.
mConfigMgr->addConfiguration("I2PBOB.cfg", mI2pSam3);
#endif
mPluginsManager->addConfigurations(mConfigMgr) ;
@ -1713,34 +1722,33 @@ int RsServer::StartupRetroShare()
{
std::cout << "RsServer::StartupRetroShare setting up hidden locations" << std::endl;
if (rsInitConfig->hiddenNodeI2PBOB) {
std::cout << "RsServer::StartupRetroShare setting up BOB" << std::endl;
if (rsInitConfig->hiddenNodeI2P) {
std::cout << "RsServer::StartupRetroShare setting up SAMv3" << std::endl;
// we need a local port!
mNetMgr->checkNetAddress();
// add i2p proxy
// bob will use this address
sockaddr_storage i2pInstance;
sockaddr_storage_ipv4_aton(i2pInstance, rsInitConfig->hiddenNodeAddress.c_str());
mPeerMgr->setProxyServerAddress(RS_HIDDEN_TYPE_I2P, i2pInstance);
std::string addr; // will be set by auto proxy service
uint16_t port = rsInitConfig->hiddenNodePort; // unused by bob
uint16_t port; // unused by SAM
bool r = autoProxy->initialSetup(autoProxyType::I2PBOB, addr, port);
bool r = autoProxy->initialSetup(autoProxyType::I2PSAM3, addr, port);
if (r && !addr.empty()) {
mPeerMgr->setupHiddenNode(addr, port);
// now enable bob
bobSettings bs;
autoProxy->taskSync(autoProxyType::I2PBOB, autoProxyTask::getSettings, &bs);
bs.enable = true;
autoProxy->taskSync(autoProxyType::I2PBOB, autoProxyTask::setSettings, &bs);
// now enable SAM
samSettings ss;
autoProxy->taskSync(autoProxyType::I2PSAM3, autoProxyTask::getSettings, &ss);
ss.enable = true;
autoProxy->taskSync(autoProxyType::I2PSAM3, autoProxyTask::setSettings, &ss);
} else {
std::cerr << "RsServer::StartupRetroShare failed to receive keys" << std::endl;
/// TODO add notify for failed bob setup
/// TODO add notify for failed i2p setup
}
} else {
mPeerMgr->setupHiddenNode(rsInitConfig->hiddenNodeAddress, rsInitConfig->hiddenNodePort);
@ -1762,19 +1770,17 @@ int RsServer::StartupRetroShare()
if (rsInitConfig->hiddenNodeSet) {
// newly created location
// mNetMgr->checkNetAddress() will setup ports for us
#if 0 // this was used for BOB but is not requires for SAMv3
// trigger updates for auto proxy services
std::vector<autoProxyType::autoProxyType_enum> types;
// i2p bob need to rebuild its command map
types.push_back(autoProxyType::I2PBOB);
rsAutoProxyMonitor::taskSync(types, autoProxyTask::reloadConfig);
#endif
}
/**************************************************************************/
/* startup (stuff dependent on Ids/peers is after this point) */
/**************************************************************************/
autoProxy->startAll();
pqih->init_listener();
@ -1807,8 +1813,8 @@ int RsServer::StartupRetroShare()
/**************************************************************************/
// auto proxy threads
#ifdef RS_USE_I2P_BOB
startServiceThread(mI2pBob, "I2P-BOB");
#ifdef RS_USE_I2P_SAM3
startServiceThread(mI2pSam3, "I2P-SAM3");
#endif
#ifdef RS_ENABLE_GXS

1117
libretroshare/src/services/autoproxy/p3i2pbob.cc
View File

File diff suppressed because it is too large Load Diff

256
libretroshare/src/services/autoproxy/p3i2pbob.h
View File

@ -1,256 +0,0 @@
/*******************************************************************************
* libretroshare/src/services/autoproxy: p3i2pbob.h *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2016 by Sehraf *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef P3I2PBOB_H
#define P3I2PBOB_H
#include <map>
#include <queue>
#include <sys/types.h>
#include "util/rstime.h"
#ifndef WINDOWS_SYS
#include <sys/socket.h>
#endif
#include "pqi/p3cfgmgr.h"
#include "services/autoproxy/rsautoproxymonitor.h"
#include "util/rsthreads.h"
#include "util/i2pcommon.h"
/*
* This class implements I2P BOB (BASIC OPEN BRIDGE) communication to allow RS
* to automatically remote control I2P to setup the needed tunnel.
* BOB is a simple text-based interface: https://geti2p.net/en/docs/api/bob
*
* Note 1:
* One tunnel is enough even for hidden locations since it can be used
* bidirectional. (In contrast to what RS I2P users had to set up manually.)
*
* Note 2:
* BOB tunnels are no SOCKS tunnel. Therefore pqissli2pbob implements a simplified
* proxy specially for BOB tunnels.
*
* Note 3:
* BOB needs a unique name as an ID for each tunnel.
* We use 'RetroShare-' + 8 random base32 characters.
*
* Design:
* The service uses three state machines to manage its task:
* int stateMachineBOB();
* mBOBState
* int stateMachineController();
* mState
* mTask
*
* stateMachineBOB:
* This state machine manages the low level communication with BOB. It basically has a linked
* list (currently a implemented as a std::map) that contains a command and the next
* state.
* Each high level operation (start up / shut down / get keys) is represented by a
* chain of states. E.g. the chain to retrieve new keys:
* mCommands[bobState::setnickN] = {setnick, bobState::newkeysN};
* mCommands[bobState::newkeysN] = {newkeys, bobState::getkeys};
* mCommands[bobState::getkeys] = {getkeys, bobState::clear};
* mCommands[bobState::clear] = {clear, bobState::quit};
* mCommands[bobState::quit] = {quit, bobState::cleared};
*
* stateMachineController:
* This state machine manages the high level tasks.
* It is controlled by mState and mTask.
*
* mTast:
* Tracks the high level operation (like start up).
* It will keep its value even when a task is done to track
* the requested BOB state.
* When other operations are performed like a conection check
* the last task gets backed up and is later restored again
*
* mState:
* This state lives only for one operation an manages the communication
* with the BOB instance. This is basically connecting, starting BOB
* protocol and disconnecting
*
* How a task looks like:
* 1) RS sets task using the ticket system
* 2) stateMachineController connects to BOBs control port, sets mBobState to a lists head
* 3) stateMachineBOB processes command chain
* 4) stateMachineBOB is done and sets mBobState to cleared signaling that the connection
* is cleared and can be closed
* 5) stateMachineController disconnects from BOBs control port and updates mState
*/
///
/// \brief The controllerState enum
/// States for the controller to keep track of what he is currently doing
enum controllerState {
csIdel,
csDoConnect,
csConnected,
csWaitForBob,
csDoDisconnect,
csDisconnected,
csError
};
///
/// \brief The controllerTask enum
/// This state tracks the controllers tast (e.g. setup a BOB tunnel or shut down
/// an existing one).
enum controllerTask {
ctIdle,
ctRunSetUp,
ctRunShutDown,
ctRunGetKeys,
ctRunCheck
};
///
/// \brief The bobState enum
/// One state for each message
///
enum bobState {
bsCleared,
bsSetnickC, // chain head for only client tunnel
bsSetnickN, // chain head for getting new (server) keys
bsSetnickS, // chain head for client and server tunnel
bsGetnick,
bsNewkeysC, // part of chain for only client tunnel
bsNewkeysN, // part of chain for getting new (server) keys
bsGetkeys,
bsSetkeys,
bsInhost,
bsOuthost,
bsInport,
bsOutport,
bsInlength,
bsOutlength,
bsInvariance,
bsOutvariance,
bsInquantity,
bsOutquantity,
bsQuiet,
bsStart,
bsStop,
bsClear,
bsList, // chain head for 'list' command
bsQuit
};
///
/// \brief The bobStateInfo struct
/// State machine with commands
/// \todo This could be replaced by a linked list instead of a map
struct bobStateInfo {
std::string command;
bobState nextState;
};
struct bobSettings : i2p::settings {};
///
/// \brief The bobStates struct
/// This container struct is used to pass all states.
/// Additionally, the tunnel name is included to to show it in the GUI.
/// The advantage of a struct is that it can be forward declared.
struct bobStates {
bobState bs;
controllerState cs;
controllerTask ct;
std::string tunnelName;
};
class p3PeerMgr;
class p3I2pBob : public RsTickingThread, public p3Config, public autoProxyService
{
public:
explicit p3I2pBob(p3PeerMgr *peerMgr);
// autoProxyService interface
public:
bool isEnabled();
bool initialSetup(std::string &addr, uint16_t &);
void processTaskAsync(taskTicket *ticket);
void processTaskSync(taskTicket *ticket);
void threadTick() override; /// @see RsTickingThread
private:
int stateMachineBOB();
int stateMachineBOB_locked_failure(const std::string &answer, const bobStateInfo &currentState);
int stateMachineController();
int stateMachineController_locked_idle();
int stateMachineController_locked_connected();
int stateMachineController_locked_disconnected();
int stateMachineController_locked_error();
// p3Config interface
protected:
RsSerialiser *setupSerialiser();
bool saveList(bool &cleanup, std::list<RsItem *> &lst);
bool loadList(std::list<RsItem *> &load);
private:
// helpers
void getBOBSettings(bobSettings *settings);
void setBOBSettings(const bobSettings *settings);
void getStates(bobStates *bs);
std::string executeCommand(const std::string &command);
bool connectI2P();
bool disconnectI2P();
void finalizeSettings_locked();
void updateSettings_locked();
std::string recv();
// states for state machines
controllerState mState;
controllerTask mTask;
// used to store old state when in error state
// mStateOld is also used as a flag when an error occured in BOB protocol
controllerState mStateOld;
// mTaskOld is used to keep the previous task (start up / shut down) when requesting keys or checking the connection
controllerTask mTaskOld;
bobSettings mSetting;
bobState mBOBState;
// used variables
p3PeerMgr *mPeerMgr;
bool mConfigLoaded;
int mSocket;
rstime_t mLastProxyCheck;
sockaddr_storage mI2PProxyAddr;
std::map<bobState, bobStateInfo> mCommands;
std::string mErrorMsg;
std::string mTunnelName;
std::queue<taskTicket *> mPending;
taskTicket *mProcessing;
// mutex
RsMutex mLock;
};
#endif // P3I2PBOB_H

798
libretroshare/src/services/autoproxy/p3i2psam3.cpp Normal file
View File

@ -0,0 +1,798 @@
#include "p3i2psam3.h"
#include <libsam3.h>
#include "pqi/p3peermgr.h"
#include "rsitems/rsconfigitems.h"
static const std::string kConfigKeySAM3Enable = "SAM3_ENABLE";
static const std::string kConfigKeyDestPriv = "DEST_PRIV";
static const std::string kConfigKeyInLength = "IN_LENGTH";
static const std::string kConfigKeyInQuantity = "IN_QUANTITY";
static const std::string kConfigKeyInVariance = "IN_VARIANCE";
static const std::string kConfigKeyInBackupQuantity = "IN_BACKUPQUANTITY";
static const std::string kConfigKeyOutLength = "OUT_LENGTH";
static const std::string kConfigKeyOutQuantity = "OUT_QUANTITY";
static const std::string kConfigKeyOutVariance = "OUT_VARIANCE";
static const std::string kConfigKeyOutBackupQuantity = "OUT_BACKUPQUANTITY";
#ifdef RS_I2P_SAM3_BOB_COMPAT
// used for migration from BOB to SAM
static const std::string kConfigKeyBOBEnable = "BOB_ENABLE";
static const std::string kConfigKeyBOBKey = "BOB_KEY";
static const std::string kConfigKeyBOBAddr = "BOB_ADDR";
#endif
static constexpr bool kDefaultSAM3Enable = false;
RS_SET_CONTEXT_DEBUG_LEVEL(2)
static void inline doSleep(std::chrono::duration<long, std::ratio<1,1000>> timeToSleepMS) {
std::this_thread::sleep_for(timeToSleepMS);
}
p3I2pSam3::p3I2pSam3(p3PeerMgr *peerMgr) :
mConfigLoaded(false), mPeerMgr(peerMgr), mPending(), mLock("p3i2p-sam3")
#ifdef RS_USE_I2P_SAM3_LIBSAM3
, mLockSam3Access("p3i2p-sam3-access")
#endif
{
RS_DBG4();
// set defaults
mSetting.initDefault();
mSetting.enable = kDefaultSAM3Enable;
mSetting.session = nullptr;
libsam3_debug = 0;
}
bool p3I2pSam3::isEnabled()
{
RS_STACK_MUTEX(mLock);
return mSetting.enable;
}
bool p3I2pSam3::initialSetup(std::string &addr, uint16_t &/*port*/)
{
RS_DBG4();
RS_STACK_MUTEX(mLock);
if (!mSetting.address.publicKey.empty() || !mSetting.address.privateKey.empty())
RS_WARN("overwriting keys!");
bool success = generateKey(mSetting.address.publicKey, mSetting.address.privateKey);
if (!success) {
RS_WARN("failed to retrieve keys");
return false;
} else {
std::string s, c;
i2p::getKeyTypes(mSetting.address.publicKey, s, c);
RS_INFO("received key ", s, " ", c);
RS_INFO("public key: ", mSetting.address.publicKey);
RS_INFO("private key: ", mSetting.address.privateKey);
RS_INFO("address: ", i2p::keyToBase32Addr(mSetting.address.publicKey));
// sanity check
auto pub = i2p::publicKeyFromPrivate(mSetting.address.privateKey);
RS_INFO("pub key derived: ", pub);
RS_INFO("address: ", i2p::keyToBase32Addr(pub));
if (pub != mSetting.address.publicKey) {
RS_WARN("public key does not match private key! fixing ...");
mSetting.address.publicKey = pub;
}
mSetting.address.base32 = i2p::keyToBase32Addr(mSetting.address.publicKey);
IndicateConfigChanged();
}
addr = mSetting.address.base32;
return true;
}
void p3I2pSam3::processTaskAsync(taskTicket *ticket)
{
RS_DBG4();
switch (ticket->task) {
case autoProxyTask::stop: [[fallthrough]];
case autoProxyTask::start: [[fallthrough]];
case autoProxyTask::receiveKey: [[fallthrough]];
case autoProxyTask::lookupKey: [[fallthrough]];
case autoProxyTask::proxyStatusCheck: [[fallthrough]];
case autoProxyTask::establishConnection: [[fallthrough]];
case autoProxyTask::closeConnection:
{
RS_STACK_MUTEX(mLock);
mPending.push(ticket);
}
break;
case autoProxyTask::status: [[fallthrough]];
case autoProxyTask::getSettings: [[fallthrough]];
case autoProxyTask::setSettings: [[fallthrough]];
case autoProxyTask::getErrorInfo: [[fallthrough]];
case autoProxyTask::reloadConfig:
// These are supposed to be sync!
RS_DBG("unknown task or sync one!");
rsAutoProxyMonitor::taskError(ticket);
break;
}
}
void p3I2pSam3::processTaskSync(taskTicket *ticket)
{
// RS_DBG4();
const bool data = !!ticket->data;
switch (ticket->task) {
case autoProxyTask::status:
{
samStatus *ss = static_cast<struct samStatus *>(ticket->data);
RS_STACK_MUTEX(mLock);
ss->state = mState;
if (mSetting.session)
ss->sessionName = mSetting.session->channel;
else
ss->sessionName = "none";
}
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
break;
case autoProxyTask::getSettings:
// check if everything needed is set
if (!data) {
RS_DBG("autoProxyTask::getSettings data is missing");
rsAutoProxyMonitor::taskError(ticket);
break;
}
// get settings
{
RS_STACK_MUTEX(mLock);
*static_cast<struct samSettings *>(ticket->data) = mSetting;
}
// finish task
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
break;
case autoProxyTask::setSettings:
// check if everything needed is set
if (!data) {
RS_DBG("autoProxyTask::setSettings data is missing");
rsAutoProxyMonitor::taskError(ticket);
break;
}
// set settings
{
RS_STACK_MUTEX(mLock);
mSetting = *static_cast<struct samSettings *>(ticket->data);
updateSettings_locked();
}
// finish task
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
break;
case autoProxyTask::getErrorInfo:
*static_cast<std::string *>(ticket->data) = mSetting.session->error;
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
break;
case autoProxyTask::reloadConfig:
{
RS_STACK_MUTEX(mLock);
updateSettings_locked();
}
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
break;
case autoProxyTask::stop:
#if 0 // doesn't seem to work, socket stays "CLOSE_WAIT"
// there can be a case where libsam3 will block forever because for some reason it does not detect that the socket it has is dead
// as a workaroung kill it from here
if (mState == samStatus::samState::connectSession || mState == samStatus::samState::connectForward) {
// lock should be held by the main thread
if (!mTmpSession) {
// now it's getting weird
RS_WARN("session is nullptr but mState says it is connecting.");
// no break! just ignore for now ...
} else {
// just close it from here, libsam3 is not thread safe.
// a bit of a hack but should do the trick
// sam3CloseSession(mSetting.session);
sam3tcpDisconnect(mTmpSession->fd);
// no break! continue as usual to keep everything in line
}
}
#endif
[[fallthrough]];
case autoProxyTask::start: [[fallthrough]];
case autoProxyTask::receiveKey: [[fallthrough]];
case autoProxyTask::lookupKey: [[fallthrough]];
case autoProxyTask::proxyStatusCheck: [[fallthrough]];
case autoProxyTask::establishConnection: [[fallthrough]];
case autoProxyTask::closeConnection:
// These are supposed to be async!
RS_WARN("unknown task or async one!");
rsAutoProxyMonitor::taskError(ticket);
break;
}
}
void p3I2pSam3::threadTick()
{
// {
// RS_STACK_MUTEX(mLock);
// Dbg4() << __PRETTY_FUNCTION__ << " mPending: " << mPending.size() << std::endl;
// }
if(mPending.empty()) {
// sleep outisde of lock!
doSleep(std::chrono::milliseconds(250));
return;
}
// get task
taskTicket *tt = nullptr;
{
RS_STACK_MUTEX(mLock);
tt = mPending.front();
mPending.pop();
}
switch (tt->task) {
case autoProxyTask::stop:
mState = samStatus::samState::offline;
stopForwarding();
stopSession();
rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::offline);
break;
case autoProxyTask::start:
{
if (!mSetting.enable) {
rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::disabled);
break;
}
// create main session
mState = samStatus::samState::connectSession;
bool ret = startSession();
if (!ret) {
mState = samStatus::samState::offline;
rsAutoProxyMonitor::taskError(tt);
break;
}
// start forwarding
mState = samStatus::samState::connectForward;
ret = startForwarding();
// finish ticket
if (ret) {
mState = samStatus::samState::online;
rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::online);
} else {
mState = samStatus::samState::offline;
rsAutoProxyMonitor::taskError(tt);
}
}
break;
case autoProxyTask::receiveKey:
{
i2p::address *addr = static_cast<i2p::address *>(tt->data);
if (generateKey(addr->publicKey, addr->privateKey)) {
addr->base32 = i2p::keyToBase32Addr(addr->publicKey);
rsAutoProxyMonitor::taskDone(tt, autoProxyStatus::ok);
} else {
rsAutoProxyMonitor::taskError(tt);
}
}
break;
case autoProxyTask::lookupKey:
lookupKey(tt);
break;
case autoProxyTask::proxyStatusCheck:
{
// TODO better detection of status
bool ok;
ok = !!mSetting.session->fd;
ok &= !!mSetting.session->fwd_fd;
*static_cast<bool*>(tt->data) = ok;
rsAutoProxyMonitor::taskDone(tt, ok ? autoProxyStatus::ok : autoProxyStatus::error);
}
break;
case autoProxyTask::establishConnection:
establishConnection(tt);
break;
case autoProxyTask::closeConnection:
closeConnection(tt);
break;
case autoProxyTask::status: [[fallthrough]];
case autoProxyTask::getSettings: [[fallthrough]];
case autoProxyTask::setSettings: [[fallthrough]];
case autoProxyTask::getErrorInfo: [[fallthrough]];
case autoProxyTask::reloadConfig:
RS_ERR("unable to handle! This is a bug! task:", tt->task);
rsAutoProxyMonitor::taskError(tt);
break;
}
tt = nullptr;
// give i2p backend some time
doSleep(std::chrono::milliseconds(100));
}
RsSerialiser *p3I2pSam3::setupSerialiser()
{
RsSerialiser* rsSerialiser = new RsSerialiser();
rsSerialiser->addSerialType(new RsGeneralConfigSerialiser());
return rsSerialiser;
}
#define addKVS(_key, _value) \
kv.key = _key;\
kv.value = _value;\
vitem->tlvkvs.pairs.push_back(kv);
#define addKVSInt(_key, _value) \
kv.key = _key;\
rs_sprintf(kv.value, "%d", _value);\
vitem->tlvkvs.pairs.push_back(kv);
bool p3I2pSam3::saveList(bool &cleanup, std::list<RsItem *> &lst)
{
RS_DBG4();
cleanup = true;
RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet;
RsTlvKeyValue kv;
RS_STACK_MUTEX(mLock);
addKVS(kConfigKeySAM3Enable, mSetting.enable ? "TRUE" : "FALSE")
addKVS(kConfigKeyDestPriv, mSetting.address.privateKey);
addKVSInt(kConfigKeyInLength, mSetting.inLength)
addKVSInt(kConfigKeyInQuantity, mSetting.inQuantity)
addKVSInt(kConfigKeyInVariance, mSetting.inVariance)
addKVSInt(kConfigKeyInBackupQuantity, mSetting.inBackupQuantity)
addKVSInt(kConfigKeyOutLength, mSetting.outLength)
addKVSInt(kConfigKeyOutQuantity, mSetting.outQuantity)
addKVSInt(kConfigKeyOutVariance, mSetting.outVariance)
addKVSInt(kConfigKeyOutBackupQuantity, mSetting.outBackupQuantity)
#ifdef RS_I2P_SAM3_BOB_COMPAT
// these allow SAMv3 users to switch back to BOB
// remove after some time
addKVS(kConfigKeyBOBEnable, mSetting.enable ? "TRUE" : "FALSE")
addKVS(kConfigKeyBOBKey, mSetting.address.privateKey)
addKVS(kConfigKeyBOBAddr, mSetting.address.base32)
#endif
lst.push_back(vitem);
return true;
}
#undef addKVS
#undef addKVSUInt
#define getKVSUInt(_kit, _key, _value) \
else if (_kit->key == _key) {\
std::istringstream is(_kit->value);\
int tmp;\
is >> tmp;\
_value = (int8_t)tmp;\
}
bool p3I2pSam3::loadList(std::list<RsItem *> &load)
{
RS_DBG4();
std::string priv;
priv.clear();
for(std::list<RsItem*>::const_iterator it = load.begin(); it!=load.end(); ++it) {
RsConfigKeyValueSet *vitem = dynamic_cast<RsConfigKeyValueSet*>(*it);
if(vitem != NULL) {
RS_STACK_MUTEX(mLock);
for(std::list<RsTlvKeyValue>::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit) {
if (kit->key == kConfigKeySAM3Enable)
mSetting.enable = kit->value == "TRUE";
else if (kit->key == kConfigKeyDestPriv)
priv = kit->value;
getKVSUInt(kit, kConfigKeyInLength, mSetting.inLength)
getKVSUInt(kit, kConfigKeyInQuantity, mSetting.inQuantity)
getKVSUInt(kit, kConfigKeyInVariance, mSetting.inVariance)
getKVSUInt(kit, kConfigKeyInBackupQuantity, mSetting.inBackupQuantity)
getKVSUInt(kit, kConfigKeyOutLength, mSetting.outLength)
getKVSUInt(kit, kConfigKeyOutQuantity, mSetting.outQuantity)
getKVSUInt(kit, kConfigKeyOutVariance, mSetting.outVariance)
getKVSUInt(kit, kConfigKeyOutBackupQuantity, mSetting.outBackupQuantity)
#ifdef RS_I2P_SAM3_BOB_COMPAT
// import BOB settings
else if (kit->key == kConfigKeyBOBEnable)
mSetting.enable = kit->value == "TRUE";
else if (kit->key == kConfigKeyBOBKey) {
// don't overwirte, just import when not set already!
if (priv.empty())
priv = kit->value;
}
#endif
else
RS_INFO("unknown key:", kit->key);
}
}
delete vitem;
}
// get the pub key
std::string pub = i2p::publicKeyFromPrivate(priv);
if (pub.empty() || priv.empty())
RS_DBG("no destination to load");
else {
RS_STACK_MUTEX(mLock);
mSetting.address.publicKey = pub;
mSetting.address.privateKey = priv;
mSetting.address.base32 = i2p::keyToBase32Addr(pub);
}
RS_STACK_MUTEX(mLock);
mConfigLoaded = true;
return true;
}
#undef getKVSUInt
bool p3I2pSam3::startSession()
{
RS_DBG4();
constexpr size_t len = 8;
const std::string location = RsRandom::alphaNumeric(len);
const std::string nick = "RetroShare-" + location;
std::vector<std::string> params;
{
RS_STACK_MUTEX(mLock);
// length
params.push_back(i2p::makeOption("inbound.length", mSetting.inLength));
params.push_back(i2p::makeOption("outbound.length", mSetting.outLength));
// variance
params.push_back(i2p::makeOption("inbound.lengthVariance", + mSetting.inVariance));
params.push_back(i2p::makeOption("outbound.lengthVariance", + mSetting.outVariance));
// quantity
params.push_back(i2p::makeOption("inbound.quantity", + mSetting.inQuantity));
params.push_back(i2p::makeOption("outbound.quantity", + mSetting.outQuantity));
// backup quantity
params.push_back(i2p::makeOption("inbound.backupQuantity", + mSetting.inBackupQuantity));
params.push_back(i2p::makeOption("outbound.backupQuantity", + mSetting.outBackupQuantity));
}
std::string paramsStr;
for (auto &&p : params)
paramsStr.append(p + " ");
// keep trailing space for easier extending when necessary
int ret;
if (mSetting.session) {
stopSession();
}
auto session = (Sam3Session*)rs_malloc(sizeof (Sam3Session));
// add nick
paramsStr.append("inbound.nickname=" + nick); // leading space is already there
{
RS_STACK_MUTEX(mLockSam3Access);
if(!mSetting.address.privateKey.empty()) {
RS_DBG3("with destination");
ret = sam3CreateSession(session, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, mSetting.address.privateKey.c_str(), Sam3SessionType::SAM3_SESSION_STREAM, Sam3SigType::EdDSA_SHA512_Ed25519, paramsStr.c_str());
} else {
RS_DBG("without destination");
ret = sam3CreateSession(session, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, SAM3_DESTINATION_TRANSIENT, Sam3SessionType::SAM3_SESSION_STREAM, Sam3SigType::EdDSA_SHA512_Ed25519, paramsStr.c_str());
}
}
if (ret != 0) {
delete session;
session = nullptr;
return false;
}
#if 0 // this check is useless. For non i2p hidden locations the public key is temporal anyway and for i2p hidden ones, it is part of the (fixed) private key.
if (!mSetting.address.publicKey.empty() && mSetting.address.publicKey != session->pubkey)
// This should be ok for non hidden locations. This should be a problem for hidden i2p locations...
RS_DBG("public key changed! Yet unsure if this is ok or a problem. Should be fine for non i2p hidden locations or clear net.");
#endif
/*
* Note: sam3CreateSession will issue a name looup of "ME" to receive its public key, thus it is always correct.
* No need to use i2p::publicKeyFromPrivate()
*/
RS_STACK_MUTEX(mLock);
mSetting.session = session;
mSetting.address.publicKey = session->pubkey;
mSetting.address.base32 = i2p::keyToBase32Addr(session->pubkey);
// do not overwrite the private key, if any!!
RS_DBG1("nick: ", nick, " address: ", mSetting.address.base32);
RS_DBG2(" myDestination.pub ", mSetting.address.publicKey);
RS_DBG2(" myDestination.priv ", mSetting.address.privateKey);
return true;
}
bool p3I2pSam3::startForwarding()
{
RS_DBG4();
if(mSetting.address.privateKey.empty()) {
RS_DBG3("no private key set");
// IMPORANT: return true here!
// since there is no forward session for non hidden nodes, this funtion is successfull by doing nothing
return true;
}
if (!mSetting.session) {
RS_WARN("no session found!");
return false;
}
peerState ps;
mPeerMgr->getOwnNetStatus(ps);
RS_STACK_MUTEX(mLockSam3Access);
mSetting.session->silent = true;
int ret = sam3StreamForward(mSetting.session, sockaddr_storage_iptostring(ps.localaddr).c_str(), sockaddr_storage_port(ps.localaddr));
if (ret < 0) {
RS_DBG("forward failed, due to", mSetting.session->error);
return false;
}
return true;
}
void p3I2pSam3::stopSession()
{
RS_DBG4();
{
RS_STACK_MUTEX(mLock);
if (!mSetting.session)
return;
// swap connections
mInvalidConnections = mValidConnections;
mValidConnections.clear();
RS_STACK_MUTEX(mLockSam3Access);
sam3CloseSession(mSetting.session);
free(mSetting.session);
mSetting.session = nullptr;
mState = samStatus::samState::offline;
}
// At least i2pd doesn't like to instantaniously stop and (re)start a session, wait here just a little bit.
// Not ideal but does the trick.
// (This happens when using the "restart" button in the settings.)
doSleep(std::chrono::seconds(10));
}
void p3I2pSam3::stopForwarding()
{
// nothing to do here, forwarding is stop when closing the seassion
}
bool p3I2pSam3::generateKey(std::string &pub, std::string &priv)
{
RS_DBG4();
pub.clear();
priv.clear();
// The session is only usef for transporting the data
Sam3Session ss;
if (0 > sam3GenerateKeys(&ss, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, Sam3SigType::EdDSA_SHA512_Ed25519)) {
RS_DBG("got error: ", ss.error);
return false;
}
pub = std::string(ss.pubkey);
priv = std::string(ss.privkey);
// sanity check
auto p = i2p::publicKeyFromPrivate(priv);
if (p != pub) {
RS_WARN("public key does not match private key! fixing ...");
pub = p;
}
RS_DBG2("publuc key / address ", pub);
RS_DBG2("private key ", priv);
return true;
}
void p3I2pSam3::lookupKey(taskTicket *ticket)
{
// this can be called independend of the main SAM session!
auto addr = static_cast<i2p::address*>(ticket->data);
if (addr->base32.empty()) {
RS_ERR("lookupKey: called with empty address");
rsAutoProxyMonitor::taskError(ticket);
return;
}
RsThread::async([ticket]()
{
auto addr = static_cast<i2p::address*>(ticket->data);
// The session is only usef for transporting the data
Sam3Session ss;
int ret = sam3NameLookup(&ss, SAM3_HOST_DEFAULT, SAM3_PORT_DEFAULT, addr->base32.c_str());
if (ret < 0) {
// get error
RS_DBG("key: ", addr->base32);
RS_DBG("got error: ", ss.error);
rsAutoProxyMonitor::taskError(ticket);
} else {
addr->publicKey = ss.destkey;
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
RS_DBG1("success");
}
});
}
void p3I2pSam3::establishConnection(taskTicket *ticket)
{
if (mState != samStatus::samState::online || !mSetting.session) {
RS_WARN("no session found!");
rsAutoProxyMonitor::taskError(ticket);
return;
}
samEstablishConnectionWrapper *wrapper = static_cast<samEstablishConnectionWrapper*>(ticket->data);
if (wrapper->address.publicKey.empty()) {
RS_ERR("no public key given");
rsAutoProxyMonitor::taskError(ticket);
return;
}
RsThread::async([ticket, this]() {
auto wrapper = static_cast<samEstablishConnectionWrapper*>(ticket->data);
struct Sam3Connection *connection;
{
auto l = this->mLockSam3Access;
RS_STACK_MUTEX(l);
mSetting.session->silent = false;
connection = sam3StreamConnect(this->mSetting.session, wrapper->address.publicKey.c_str());
}
if (!connection) {
// get error
RS_DBG("got error:", this->mSetting.session->error);
rsAutoProxyMonitor::taskError(ticket);
} else {
wrapper->connection = connection;
{
auto l = this->mLockSam3Access;
RS_STACK_MUTEX(l);
this->mValidConnections.push_back(connection);
}
RS_DBG1("success");
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
}
});
}
void p3I2pSam3::closeConnection(taskTicket *ticket)
{
Sam3Connection *conn = static_cast<Sam3Connection*>(ticket->data);
if (mState == samStatus::samState::offline || !mSetting.session) {
// no session found, sam was likely stopped
RS_DBG2("no session found");
auto it = std::find(mInvalidConnections.begin(), mInvalidConnections.end(), conn);
if (it != mInvalidConnections.end()) {
// this is the expected case
mInvalidConnections.erase(it);
} else {
// this is unexpected but not a big deal, just warn
RS_WARN("cannot find connection in mInvalidConnections");
it = std::find(mValidConnections.begin(), mValidConnections.end(), conn);
if (it != mValidConnections.end()) {
mValidConnections.erase(it);
// now it is getting even weirder, still not a big deal, just warn
RS_WARN("found connection in mValidConnections");
}
}
// when libsam3 has already handled closing of the connection - which should be the case here - the memory has been freed already (-> pointer is invalid)
conn = nullptr;
} else {
RS_STACK_MUTEX(mLock);
bool callClose = true;
// search in current connections
auto it = std::find(mValidConnections.begin(), mValidConnections.end(), conn);
if (it != mValidConnections.end()) {
RS_DBG2("found valid connection");
mValidConnections.erase(it);
} else {
// search in old connections
it = std::find(mInvalidConnections.begin(), mInvalidConnections.end(), conn);
if (it != mInvalidConnections.end()) {
// old connection, just ignore. *should* be freed already
mInvalidConnections.erase(it);
RS_DBG2("found old (invalid) connection");
callClose = false;
conn = nullptr;
} else {
// weird
RS_WARN("could'n find connection!");
// best thing we can do here
callClose = false;
conn = nullptr;
}
}
if (callClose) {
RS_DBG2("closing connection");
RS_STACK_MUTEX(mLockSam3Access);
sam3CloseConnection(conn);
conn = nullptr; // freed by above call
}
}
if (conn) {
free(conn);
conn = nullptr;
}
ticket->data = nullptr;
rsAutoProxyMonitor::taskDone(ticket, autoProxyStatus::ok);
return;
}
void p3I2pSam3::updateSettings_locked()
{
RS_DBG4();
IndicateConfigChanged();
#if 0 // TODO recreat session when active, can we just recreat it?
if (mSs) {
stopSession();
startSession();
}
#endif
}

110
libretroshare/src/services/autoproxy/p3i2psam3.h Normal file
View File

@ -0,0 +1,110 @@
#ifndef P3I2PSAM3_H
#define P3I2PSAM3_H
#include <queue>
#include <list>
#include "services/autoproxy/rsautoproxymonitor.h"
#include "pqi/p3cfgmgr.h"
#include "util/i2pcommon.h"
#include "util/rsthreads.h"
/*
* This class implements I2P SAMv3 (Simple Anonymous Messaging) to allow RS
* to automatically setup tunnel to and from I2P.
* SAMv3 is a simple text-based interface: https://geti2p.net/de/docs/api/samv3
*
* For the actual SAM commands / low level stuff libsam3 (https://github.com/i2p/libsam3)
* is used with some minor adjustments, for exmaple, the FORWARD session is always silent.
*
* SAM in a nutshell works like this:
* 1) setup main/control session which configures everything (destination ID, tunnel number, hops number, and so on)
* 2) setup a forward session, so that I2P will establish a connection to RS for each incoming connection to our i2p destination
* 3a) query/lookup the destination (public key) for a given i2p address
* 3b) connect to the given destination
*
* An established connection (both incoming or outgoing) are then handed over to RS.
* The lifetime of a session (and its subordinates connections) is bound to their tcp socket. When the socket closes, the session is closed, too.
*
*/
class p3PeerMgr;
class Sam3Session;
class Sam3Connection;
typedef Sam3Session samSession;
struct samSettings : i2p::settings {
samSession *session;
};
struct samEstablishConnectionWrapper {
i2p::address address;
Sam3Connection *connection;
};
struct samStatus {
std::string sessionName;
enum samState {
offline,
connectSession,
connectForward,
online
} state; // the name is kinda redundant ...
};
class p3I2pSam3 : public RsTickingThread, public p3Config, public autoProxyService
{
public:
p3I2pSam3(p3PeerMgr *peerMgr);
// autoProxyService interface
public:
bool isEnabled();
bool initialSetup(std::string &addr, uint16_t &port);
void processTaskAsync(taskTicket *ticket);
void processTaskSync(taskTicket *ticket);
// RsTickingThread interface
public:
void threadTick(); /// @see RsTickingThread
// p3Config interface
protected:
RsSerialiser *setupSerialiser();
bool saveList(bool &cleanup, std::list<RsItem *> &);
bool loadList(std::list<RsItem *> &load);
private:
bool startSession();
bool startForwarding();
void stopSession();
void stopForwarding();
bool generateKey(std::string &pub, std::string &priv);
void lookupKey(taskTicket *ticket);
void establishConnection(taskTicket *ticket);
void closeConnection(taskTicket *ticket);
void updateSettings_locked();
bool mConfigLoaded;
samSettings mSetting;
p3PeerMgr *mPeerMgr;
std::queue<taskTicket *> mPending;
// Used to report the state to the gui
// (Since the create session call/will can block and there is no easy way from outside the main thread to see
// what is going on, it is easier to store the current state in an extra variable independen from the main thread)
samStatus::samState mState;
// used to keep track of connections, libsam3 does it internally but it can be unreliable since pointers are shared
std::list<Sam3Connection *> mValidConnections, mInvalidConnections;
// mutex
RsMutex mLock;
RsMutex mLockSam3Access; // libsam3 is not thread safe! (except for key lookup)
};
#endif // P3I2PSAM3_H

20
libretroshare/src/services/autoproxy/rsautoproxymonitor.cc
View File

@ -329,14 +329,22 @@ autoProxyService *rsAutoProxyMonitor::lookUpService(autoProxyType::autoProxyType
bool rsAutoProxyMonitor::isAsyncTask(autoProxyTask::autoProxyTask_enum t)
{
// Explicit list all values, so that missing ones will be detected by the compiler.
switch (t) {
case autoProxyTask::start:
case autoProxyTask::stop:
case autoProxyTask::receiveKey:
case autoProxyTask::start: [[fallthrough]];
case autoProxyTask::stop: [[fallthrough]];
case autoProxyTask::receiveKey: [[fallthrough]];
case autoProxyTask::lookupKey: [[fallthrough]];
case autoProxyTask::establishConnection: [[fallthrough]];
case autoProxyTask::closeConnection:
return true;
break;
default:
break;
case autoProxyTask::status: [[fallthrough]];
case autoProxyTask::getSettings: [[fallthrough]];
case autoProxyTask::setSettings: [[fallthrough]];
case autoProxyTask::getErrorInfo: [[fallthrough]];
case autoProxyTask::reloadConfig: [[fallthrough]];
case autoProxyTask::proxyStatusCheck:
return false;
}
return false;
}

24
libretroshare/src/services/autoproxy/rsautoproxymonitor.h
View File

@ -31,23 +31,27 @@ class autoProxyCallback;
namespace autoProxyType {
enum autoProxyType_enum {
I2PBOB
// I2PBOB,
I2PSAM3
};
}
namespace autoProxyTask {
enum autoProxyTask_enum {
/* async tasks */
start, ///< start up proxy
stop, ///< shut down proxy
receiveKey, ///< renew proxy key (if any)
proxyStatusCheck, ///< use to check if the proxy is still running
start, ///< start up proxy
stop, ///< shut down proxy
receiveKey, ///< renew proxy key (if any)
lookupKey, ///< look up a base32 addr
proxyStatusCheck, ///< use to check if the proxy is still running
establishConnection, ///< create a connection to a given public key or base32 address
closeConnection, ///< closes a connection
/* sync tasks */
status, ///< get status from auto proxy
getSettings, ///< get setting from auto proxy
setSettings, ///< set setting of auto proxy
reloadConfig, ///< signal config reload/rebuild
getErrorInfo ///< get error information from auto proxy
status, ///< get status from auto proxy
getSettings, ///< get setting from auto proxy
setSettings, ///< set setting of auto proxy
reloadConfig, ///< signal config reload/rebuild
getErrorInfo ///< get error information from auto proxy
};
}

686
libretroshare/src/services/p3gxsforums.cc
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -59,7 +59,11 @@ p3GxsForums::p3GxsForums( RsGeneralDataService *gds,
RsGenExchange( gds, nes, new RsGxsForumSerialiser(),
RS_SERVICE_GXS_TYPE_FORUMS, gixs, forumsAuthenPolicy()),
RsGxsForums(static_cast<RsGxsIface&>(*this)), mGenToken(0),
mGenActive(false), mGenCount(0), mKnownForumsMutex("GXS forums known forums timestamp cache")
mGenActive(false), mGenCount(0),
mKnownForumsMutex("GXS forums known forums timestamp cache")
#ifdef RS_DEEP_FORUMS_INDEX
, mDeepIndex(DeepForumsIndex::dbDefaultPath())
#endif
{
// Test Data disabled in Repo.
//RsTickEvent::schedule_in(FORUM_TESTEVENT_DUMMYDATA, DUMMYDATA_PERIOD);
@ -190,223 +194,254 @@ RsSerialiser* p3GxsForums::setupSerialiser()
return rss;
}
void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
void p3GxsForums::notifyChanges(std::vector<RsGxsNotify*>& changes)
{
#ifdef GXSFORUMS_DEBUG
std::cerr << "p3GxsForums::notifyChanges() : " << changes.size() << "changes to notify" << std::endl;
#endif
RS_DBG2(changes.size(), " changes to notify");
std::vector<RsGxsNotify *>::iterator it;
for(it = changes.begin(); it != changes.end(); ++it)
for(RsGxsNotify* gxsChange: changes)
{
RsGxsMsgChange *msgChange = dynamic_cast<RsGxsMsgChange *>(*it);
// Let the compiler delete the change for us
std::unique_ptr<RsGxsNotify> gxsChangeDeleter(gxsChange);
if (msgChange)
switch(gxsChange->getType())
{
if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW || msgChange->getType() == RsGxsNotify::TYPE_PUBLISHED) /* message received */
if (rsEvents)
case RsGxsNotify::TYPE_RECEIVED_NEW: // [[fallthrough]]
case RsGxsNotify::TYPE_PUBLISHED:
{
auto msgChange = dynamic_cast<RsGxsMsgChange*>(gxsChange);
if(msgChange) /* Message received */
{
uint8_t msgSubtype = msgChange->mNewMsgItem->PacketSubType();
switch(static_cast<RsGxsForumsItems>(msgSubtype))
{
case RsGxsForumsItems::MESSAGE_ITEM:
{
auto newForumMessageItem =
dynamic_cast<RsGxsForumMsgItem*>(
msgChange->mNewMsgItem );
if(!newForumMessageItem)
{
RS_ERR("Received message change with mNewMsgItem type "
"mismatching or null");
print_stacktrace();
return;
}
#ifdef RS_DEEP_FORUMS_INDEX
RsGxsForumMsg tmpPost = newForumMessageItem->mMsg;
tmpPost.mMeta = newForumMessageItem->meta;
mDeepIndex.indexForumPost(tmpPost);
#endif
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumMsgId = msgChange->mMsgId;
ev->mForumGroupId = msgChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::NEW_MESSAGE;
rsEvents->postEvent(ev);
break;
}
#ifdef NOT_USED_YET
if (!msgChange->metaChange())
{
#ifdef GXSCHANNELS_DEBUG
std::cerr << "p3GxsForums::notifyChanges() Found Message Change Notification";
std::cerr << std::endl;
#endif
std::map<RsGxsGroupId, std::set<RsGxsMessageId> > &msgChangeMap = msgChange->msgChangeMap;
for(auto mit = msgChangeMap.begin(); mit != msgChangeMap.end(); ++mit)
{
#ifdef GXSCHANNELS_DEBUG
std::cerr << "p3GxsForums::notifyChanges() Msgs for Group: " << mit->first;
std::cerr << std::endl;
#endif
bool enabled = false;
if (autoDownloadEnabled(mit->first, enabled) && enabled)
{
#ifdef GXSCHANNELS_DEBUG
std::cerr << "p3GxsChannels::notifyChanges() AutoDownload for Group: " << mit->first;
std::cerr << std::endl;
#endif
/* problem is most of these will be comments and votes,
* should make it occasional - every 5mins / 10minutes TODO */
unprocessedGroups.push_back(mit->first);
}
default:
RS_WARN("Got unknown gxs message subtype: ", msgSubtype);
break;
}
}
#endif
auto groupChange = dynamic_cast<RsGxsGroupChange*>(gxsChange);
if(groupChange) /* Group received */
{
bool unknown;
{
RS_STACK_MUTEX(mKnownForumsMutex);
unknown = ( mKnownForums.find(gxsChange->mGroupId)
== mKnownForums.end() );
mKnownForums[gxsChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
}
if(unknown)
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::NEW_FORUM;
rsEvents->postEvent(ev);
}
#ifdef RS_DEEP_FORUMS_INDEX
uint8_t itemType = groupChange->mNewGroupItem->PacketSubType();
switch(static_cast<RsGxsForumsItems>(itemType))
{
case RsGxsForumsItems::GROUP_ITEM:
{
auto newForumGroupItem =
static_cast<RsGxsForumGroupItem*>(
groupChange->mNewGroupItem );
mDeepIndex.indexForumGroup(newForumGroupItem->mGroup);
break;
}
default:
RS_WARN("Got unknown gxs group subtype: ", itemType);
break;
}
#endif // def RS_DEEP_FORUMS_INDEX
}
break;
}
else
case RsGxsNotify::TYPE_PROCESSED: // happens when the group is subscribed
{
if (rsEvents)
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED;
rsEvents->postEvent(ev);
break;
}
case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED;
rsEvents->postEvent(ev);
break;
}
case RsGxsNotify::TYPE_MESSAGE_DELETED:
{
auto delChange = dynamic_cast<RsGxsMsgDeletedChange*>(gxsChange);
if(!delChange)
{
RsGxsGroupChange *grpChange = dynamic_cast<RsGxsGroupChange*>(*it);
if (grpChange)
{
switch (grpChange->getType())
{
case RsGxsNotify::TYPE_PROCESSED: // happens when the group is subscribed
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED;
rsEvents->postEvent(ev);
}
break;
case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED;
rsEvents->postEvent(ev);
}
break;
case RsGxsNotify::TYPE_PUBLISHED:
case RsGxsNotify::TYPE_RECEIVED_NEW:
{
/* group received */
bool unknown;
{
RS_STACK_MUTEX(mKnownForumsMutex);
unknown = (mKnownForums.find(grpChange->mGroupId)==mKnownForums.end());
mKnownForums[grpChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
}
if(unknown)
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::NEW_FORUM;
rsEvents->postEvent(ev);
}
else
RsInfo() << __PRETTY_FUNCTION__
<< " Not notifying already known forum "
<< grpChange->mGroupId << std::endl;
}
break;
case RsGxsNotify::TYPE_GROUP_DELETED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::DELETED_FORUM;
rsEvents->postEvent(ev);
}
break;
case RsGxsNotify::TYPE_STATISTICS_CHANGED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = grpChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED;
rsEvents->postEvent(ev);
RS_STACK_MUTEX(mKnownForumsMutex);
mKnownForums[grpChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
}
break;
case RsGxsNotify::TYPE_UPDATED:
{
// Happens when the group data has changed. In this case we need to analyse the old and new group in order to detect possible notifications for clients
RsGxsForumGroupItem *old_forum_grp_item = dynamic_cast<RsGxsForumGroupItem*>(grpChange->mOldGroupItem);
RsGxsForumGroupItem *new_forum_grp_item = dynamic_cast<RsGxsForumGroupItem*>(grpChange->mNewGroupItem);
if(old_forum_grp_item == nullptr || new_forum_grp_item == nullptr)
{
RsErr() << __PRETTY_FUNCTION__ << " received GxsGroupUpdate item with mOldGroup and mNewGroup not of type RsGxsForumGroupItem or NULL. This is inconsistent!" << std::endl;
delete grpChange;
continue;
}
// First of all, we check if there is a difference between the old and new list of moderators
std::list<RsGxsId> added_mods, removed_mods;
for(auto& gxs_id: new_forum_grp_item->mGroup.mAdminList.ids)
if(old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == old_forum_grp_item->mGroup.mAdminList.ids.end())
added_mods.push_back(gxs_id);
for(auto& gxs_id: old_forum_grp_item->mGroup.mAdminList.ids)
if(new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == new_forum_grp_item->mGroup.mAdminList.ids.end())
removed_mods.push_back(gxs_id);
if(!added_mods.empty() || !removed_mods.empty())
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mModeratorsAdded = added_mods;
ev->mModeratorsRemoved = removed_mods;
ev->mForumEventCode = RsForumEventCode::MODERATOR_LIST_CHANGED;
rsEvents->postEvent(ev);
}
// check the list of pinned posts
std::list<RsGxsMessageId> added_pins, removed_pins;
for(auto& msg_id: new_forum_grp_item->mGroup.mPinnedPosts.ids)
if(old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == old_forum_grp_item->mGroup.mPinnedPosts.ids.end())
added_pins.push_back(msg_id);
for(auto& msg_id: old_forum_grp_item->mGroup.mPinnedPosts.ids)
if(new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == new_forum_grp_item->mGroup.mPinnedPosts.ids.end())
removed_pins.push_back(msg_id);
if(!added_pins.empty() || !removed_pins.empty())
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mForumEventCode = RsForumEventCode::PINNED_POSTS_CHANGED;
rsEvents->postEvent(ev);
}
if( old_forum_grp_item->mGroup.mDescription != new_forum_grp_item->mGroup.mDescription
|| old_forum_grp_item->meta.mGroupName != new_forum_grp_item->meta.mGroupName
|| old_forum_grp_item->meta.mGroupFlags != new_forum_grp_item->meta.mGroupFlags
|| old_forum_grp_item->meta.mAuthorId != new_forum_grp_item->meta.mAuthorId
|| old_forum_grp_item->meta.mCircleId != new_forum_grp_item->meta.mCircleId
)
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mForumEventCode = RsForumEventCode::UPDATED_FORUM;
rsEvents->postEvent(ev);
}
}
break;
default:
RsErr() << " Got a GXS event of type " << grpChange->getType() << " Currently not handled." << std::endl;
break;
}
}
RS_ERR( "Got mismatching notification type: ",
gxsChange->getType() );
print_stacktrace();
break;
}
#ifdef RS_DEEP_FORUMS_INDEX
mDeepIndex.removeForumPostFromIndex(
delChange->mGroupId, delChange->messageId);
#endif
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumEventCode = RsForumEventCode::DELETED_POST;
ev->mForumGroupId = delChange->mGroupId;
ev->mForumMsgId = delChange->messageId;
break;
}
case RsGxsNotify::TYPE_GROUP_DELETED:
{
#ifdef RS_DEEP_FORUMS_INDEX
mDeepIndex.removeForumFromIndex(gxsChange->mGroupId);
#endif
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::DELETED_FORUM;
rsEvents->postEvent(ev);
break;
}
case RsGxsNotify::TYPE_STATISTICS_CHANGED:
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = gxsChange->mGroupId;
ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED;
rsEvents->postEvent(ev);
RS_STACK_MUTEX(mKnownForumsMutex);
mKnownForums[gxsChange->mGroupId] = time(nullptr);
IndicateConfigChanged();
break;
}
case RsGxsNotify::TYPE_UPDATED:
{
/* Happens when the group data has changed. In this case we need to
* analyse the old and new group in order to detect possible
* notifications for clients */
auto grpChange = dynamic_cast<RsGxsGroupChange*>(gxsChange);
RsGxsForumGroupItem* old_forum_grp_item =
dynamic_cast<RsGxsForumGroupItem*>(grpChange->mOldGroupItem);
RsGxsForumGroupItem* new_forum_grp_item =
dynamic_cast<RsGxsForumGroupItem*>(grpChange->mNewGroupItem);
if( old_forum_grp_item == nullptr || new_forum_grp_item == nullptr)
{
RS_ERR( "received GxsGroupUpdate item with mOldGroup and "
"mNewGroup not of type RsGxsForumGroupItem or NULL. "
"This is inconsistent!");
print_stacktrace();
break;
}
#ifdef RS_DEEP_FORUMS_INDEX
mDeepIndex.indexForumGroup(new_forum_grp_item->mGroup);
#endif
/* First of all, we check if there is a difference between the old
* and new list of moderators */
std::list<RsGxsId> added_mods, removed_mods;
for(auto& gxs_id: new_forum_grp_item->mGroup.mAdminList.ids)
if( old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id)
== old_forum_grp_item->mGroup.mAdminList.ids.end() )
added_mods.push_back(gxs_id);
for(auto& gxs_id: old_forum_grp_item->mGroup.mAdminList.ids)
if( new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id)
== new_forum_grp_item->mGroup.mAdminList.ids.end() )
removed_mods.push_back(gxs_id);
if(!added_mods.empty() || !removed_mods.empty())
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mModeratorsAdded = added_mods;
ev->mModeratorsRemoved = removed_mods;
ev->mForumEventCode = RsForumEventCode::MODERATOR_LIST_CHANGED;
rsEvents->postEvent(ev);
}
// check the list of pinned posts
std::list<RsGxsMessageId> added_pins, removed_pins;
for(auto& msg_id: new_forum_grp_item->mGroup.mPinnedPosts.ids)
if( old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id)
== old_forum_grp_item->mGroup.mPinnedPosts.ids.end() )
added_pins.push_back(msg_id);
for(auto& msg_id: old_forum_grp_item->mGroup.mPinnedPosts.ids)
if( new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id)
== new_forum_grp_item->mGroup.mPinnedPosts.ids.end() )
removed_pins.push_back(msg_id);
if(!added_pins.empty() || !removed_pins.empty())
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mForumEventCode = RsForumEventCode::PINNED_POSTS_CHANGED;
rsEvents->postEvent(ev);
}
if( old_forum_grp_item->mGroup.mDescription != new_forum_grp_item->mGroup.mDescription
|| old_forum_grp_item->meta.mGroupName != new_forum_grp_item->meta.mGroupName
|| old_forum_grp_item->meta.mGroupFlags != new_forum_grp_item->meta.mGroupFlags
|| old_forum_grp_item->meta.mAuthorId != new_forum_grp_item->meta.mAuthorId
|| old_forum_grp_item->meta.mCircleId != new_forum_grp_item->meta.mCircleId )
{
auto ev = std::make_shared<RsGxsForumEvent>();
ev->mForumGroupId = new_forum_grp_item->meta.mGroupId;
ev->mForumEventCode = RsForumEventCode::UPDATED_FORUM;
rsEvents->postEvent(ev);
}
break;
}
/* shouldn't need to worry about groups - as they need to be subscribed to */
delete *it;
default:
RS_ERR( "Got a GXS event of type ", gxsChange->getType(),
" Currently not handled." );
break;
}
}
}
@ -618,6 +653,8 @@ bool p3GxsForums::createForumV2(
forum.mMeta.mSignFlags = GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_NONEREQ
| GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_REQUIRED;
/* This flag have always this value even for circle restricted forums due to
* how GXS distribute/verify groups */
forum.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC;
forum.mMeta.mCircleId.clear();
@ -1370,6 +1407,255 @@ bool RsGxsForumGroup::canEditPosts(const RsGxsId& id) const
id == mMeta.mAuthorId;
}
std::error_condition p3GxsForums::getContentSummaries(
const RsGxsGroupId& forumId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries )
{
uint32_t token;
RsTokReqOptions opts;
opts.mReqType = GXS_REQUEST_TYPE_MSG_META;
GxsMsgReq msgReq;
msgReq[forumId] = contentIds;
if(!requestMsgInfo(token, opts, msgReq))
{
RS_ERR("requestMsgInfo failed");
return std::errc::invalid_argument;
}
switch(waitToken(token, std::chrono::seconds(5)))
{
case RsTokenService::COMPLETE:
{
GxsMsgMetaMap metaMap;
if(!RsGenExchange::getMsgMeta(token, metaMap))
return std::errc::result_out_of_range;
summaries = metaMap[forumId];
return std::error_condition();
}
case RsTokenService::PARTIAL: // [[fallthrough]];
case RsTokenService::PENDING:
return std::errc::timed_out;
default:
return std::errc::not_supported;
}
}
#ifdef RS_DEEP_FORUMS_INDEX
std::error_condition p3GxsForums::handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
RS_DBG1("");
RsGxsForumsSearchRequest request;
{
RsGenericSerializer::SerializeContext ctx(requestData, requestSize);
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::DESERIALIZE;
RS_SERIAL_PROCESS(request);
}
if(request.mType != RsGxsForumsItems::SEARCH_REQUEST)
{
// If more types are implemented we would put a switch on mType instead
RS_WARN( "Got search request with unkown type: ",
static_cast<uint32_t>(request.mType) );
return std::errc::bad_message;
}
RsGxsForumsSearchReply reply;
auto mErr = prepareSearchResults(request.mQuery, true, reply.mResults);
if(mErr || reply.mResults.empty()) return mErr;
{
RsGenericSerializer::SerializeContext ctx;
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::SIZE_ESTIMATE;
RS_SERIAL_PROCESS(reply);
resultSize = ctx.mOffset;
}
resultData = rs_malloc<uint8_t>(resultSize);
RsGenericSerializer::SerializeContext ctx(resultData, resultSize);
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::SERIALIZE;
RS_SERIAL_PROCESS(reply);
return std::error_condition();
}
std::error_condition p3GxsForums::distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId )
{
RsGxsForumsSearchRequest request;
request.mQuery = matchString;
uint32_t requestSize;
{
RsGenericSerializer::SerializeContext ctx;
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::SIZE_ESTIMATE;
RS_SERIAL_PROCESS(request);
requestSize = ctx.mOffset;
}
std::error_condition ec;
auto requestData = rs_malloc<uint8_t>(requestSize, &ec);
if(!requestData) return ec;
{
RsGenericSerializer::SerializeContext ctx(requestData, requestSize);
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::SERIALIZE;
RS_SERIAL_PROCESS(request);
}
return netService()->distantSearchRequest(
requestData, requestSize,
static_cast<RsServiceType>(serviceType()), searchId );
}
std::error_condition p3GxsForums::localSearch(
const std::string& matchString,
std::vector<RsGxsSearchResult>& searchResults )
{ return prepareSearchResults(matchString, false, searchResults); }
std::error_condition p3GxsForums::prepareSearchResults(
const std::string& matchString, bool publicOnly,
std::vector<RsGxsSearchResult>& searchResults )
{
std::vector<DeepForumsSearchResult> results;
auto mErr = mDeepIndex.search(matchString, results);
if(mErr) return mErr;
searchResults.clear();
for(auto uRes: results)
{
RsUrl resUrl(uRes.mUrl);
const auto forumIdStr = resUrl.getQueryV(RsGxsForums::FORUM_URL_ID_FIELD);
if(!forumIdStr)
{
RS_ERR( "Forum URL retrieved from deep index miss ID. ",
"Should never happen! ", uRes.mUrl );
print_stacktrace();
return std::errc::address_not_available;
}
std::vector<RsGxsForumGroup> forumsInfo;
RsGxsGroupId forumId(*forumIdStr);
if(forumId.isNull())
{
RS_ERR( "Forum ID retrieved from deep index is invalid. ",
"Should never happen! ", uRes.mUrl );
print_stacktrace();
return std::errc::bad_address;
}
if( !getForumsInfo(std::list<RsGxsGroupId>{forumId}, forumsInfo) ||
forumsInfo.empty() )
{
RS_ERR( "Forum just parsed from deep index link not found. "
"Should never happen! ", forumId, " ", uRes.mUrl );
print_stacktrace();
return std::errc::identifier_removed;
}
RsGroupMetaData& fMeta(forumsInfo[0].mMeta);
// Avoid leaking sensitive information to unkown peers
if( publicOnly &&
( static_cast<RsGxsCircleType>(fMeta.mCircleType) !=
RsGxsCircleType::PUBLIC ) ) continue;
RsGxsSearchResult res;
res.mGroupId = forumId;
res.mGroupName = fMeta.mGroupName;
res.mAuthorId = fMeta.mAuthorId;
res.mPublishTs = fMeta.mPublishTs;
res.mSearchContext = uRes.mSnippet;
auto postIdStr =
resUrl.getQueryV(RsGxsForums::FORUM_URL_MSG_ID_FIELD);
if(postIdStr)
{
RsGxsMessageId msgId(*postIdStr);
if(msgId.isNull())
{
RS_ERR( "Post just parsed from deep index link is invalid. "
"Should never happen! ", postIdStr, " ", uRes.mUrl );
print_stacktrace();
return std::errc::bad_address;
}
std::vector<RsMsgMetaData> msgSummaries;
auto errc = getContentSummaries(
forumId, std::set<RsGxsMessageId>{msgId}, msgSummaries);
if(errc) return errc;
if(msgSummaries.size() != 1)
{
RS_ERR( "getContentSummaries returned: ", msgSummaries.size(),
"should never happen!" );
return std::errc::result_out_of_range;
}
RsMsgMetaData& msgMeta(msgSummaries[0]);
res.mMsgId = msgMeta.mMsgId;
res.mMsgName = msgMeta.mMsgName;
res.mAuthorId = msgMeta.mAuthorId;
}
RS_DBG4(res);
searchResults.push_back(res);
}
return std::error_condition();
}
std::error_condition p3GxsForums::receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize )
{
RsGxsForumsSearchReply reply;
{
RsGenericSerializer::SerializeContext ctx(resultData, resultSize);
RsGenericSerializer::SerializeJob j =
RsGenericSerializer::SerializeJob::DESERIALIZE;
RS_SERIAL_PROCESS(reply);
}
free(resultData);
if(reply.mType != RsGxsForumsItems::SEARCH_REPLY)
{
// If more types are implemented we would put a switch on mType instead
RS_WARN( "Got search request with unkown type: ",
static_cast<uint32_t>(reply.mType) );
return std::errc::bad_message;
}
auto event = std::make_shared<RsGxsForumsDistantSearchEvent>();
event->mSearchId = requestId;
event->mSearchResults = reply.mResults;
rsEvents->postEvent(event);
return std::error_condition();
}
#else // def RS_DEEP_FORUMS_INDEX
std::error_condition p3GxsForums::distantSearchRequest(
const std::string&, TurtleRequestId& )
{ return std::errc::function_not_supported; }
std::error_condition p3GxsForums::localSearch(
const std::string&,
std::vector<RsGxsSearchResult>& )
{ return std::errc::function_not_supported; }
#endif // def RS_DEEP_FORUMS_INDEX
/*static*/ const std::string RsGxsForums::DEFAULT_FORUM_BASE_URL =
"retroshare:///forums";
/*static*/ const std::string RsGxsForums::FORUM_URL_NAME_FIELD =

52
libretroshare/src/services/p3gxsforums.h
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2012-2014 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -32,6 +32,10 @@
#include "util/rstickevent.h"
#include "util/rsdebug.h"
#ifdef RS_DEEP_FORUMS_INDEX
#include "deep_search/forumsindex.hpp"
#endif
class p3GxsForums: public RsGenExchange, public RsGxsForums, public p3Config,
public RsTickEvent /* only needed for testing - remove after */
@ -142,7 +146,34 @@ public:
/// @see RsGxsForums
std::error_condition setPostKeepForever(
const RsGxsGroupId& forumId, const RsGxsMessageId& postId,
bool keepForever ) override;
bool keepForever ) override;
/// @see RsGxsForums
std::error_condition getContentSummaries(
const RsGxsGroupId& forumId,
const std::set<RsGxsMessageId>& contentIds,
std::vector<RsMsgMetaData>& summaries ) override;
/// @see RsGxsForums
std::error_condition distantSearchRequest(
const std::string& matchString, TurtleRequestId& searchId ) override;
/// @see RsGxsForums
std::error_condition localSearch(
const std::string& matchString,
std::vector<RsGxsSearchResult>& searchResults ) override;
#ifdef RS_DEEP_FORUMS_INDEX
/// @see RsNxsObserver
std::error_condition handleDistantSearchRequest(
rs_view_ptr<uint8_t> requestData, uint32_t requestSize,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
/// @see RsNxsObserver
std::error_condition receiveDistantSearchResult(
const TurtleRequestId requestId,
rs_owner_ptr<uint8_t>& resultData, uint32_t& resultSize ) override;
#endif
/// implementation of rsGxsGorums
///
@ -155,6 +186,17 @@ public:
bool getMsgMetaData(const uint32_t &token, GxsMsgMetaMap& msg_metas) ;
protected:
#ifdef RS_DEEP_FORUMS_INDEX
/** Internal usage
* @param[in] publicOnly if true is passed only results pertaining to
* publicly shared forums are returned
*/
std::error_condition prepareSearchResults(
const std::string& matchString, bool publicOnly,
std::vector<RsGxsSearchResult>& searchResults );
#endif //def RS_DEEP_FORUMS_INDEX
private:
static uint32_t forumsAuthenPolicy();
@ -189,4 +231,8 @@ bool generateGroup(uint32_t &token, std::string groupName);
std::map<RsGxsGroupId,rstime_t> mKnownForums ;
RsMutex mKnownForumsMutex;
#ifdef RS_DEEP_FORUMS_INDEX
DeepForumsIndex mDeepIndex;
#endif
};

37
libretroshare/src/turtle/p3turtle.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2009-2018 by Cyril Soler <csoler@users.sourceforge.net> *
* Copyright (C) 2009-2018 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright (C) 2018-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -26,6 +28,10 @@
#include <stdexcept>
#include <stdlib.h>
#include <assert.h>
#include <iostream>
#include <cerrno>
#include <cmath>
#include <cstdio>
#include "rsserver/p3face.h"
#include "crypto/rscrypto.h"
@ -39,13 +45,7 @@
#include "ft/ftcontroller.h"
#include "p3turtle.h"
#include <iostream>
#include <errno.h>
#include <cmath>
#include <stdio.h>
#include "util/cxx17retrocompat.h"
#include "util/rsdebug.h"
#include "util/rsprint.h"
#include "util/rsrandom.h"
@ -1975,7 +1975,8 @@ void p3turtle::handleTunnelResult(RsTurtleTunnelOkItem *item)
//
void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result) const
void RsTurtleStringSearchRequestItem::search(
std::list<TurtleFileInfo>& result ) const
{
/* call to core */
std::list<DirDetails> initialResults;
@ -1988,17 +1989,19 @@ void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result)
std::cerr << "Performing rsFiles->search()" << std::endl ;
#endif
// now, search!
rsFiles->SearchKeywords(words, initialResults,RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SEARCHABLE,PeerId());
rsFiles->SearchKeywords(
words, initialResults,
RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SEARCHABLE, PeerId() );
#ifdef P3TURTLE_DEBUG
std::cerr << initialResults.size() << " matches found." << std::endl ;
#endif
result.clear() ;
for(std::list<DirDetails>::const_iterator it(initialResults.begin());it!=initialResults.end();++it)
for(auto& it: std::as_const(initialResults))
{
// retain only file type
if (it->type == DIR_TYPE_DIR)
if (it.type == DIR_TYPE_DIR)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Skipping directory " << it->name << std::endl ;
@ -2006,12 +2009,12 @@ void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result)
continue;
}
TurtleFileInfo i ;
i.hash = it->hash ;
i.size = it->size ;
i.name = it->name ;
TurtleFileInfo i;
i.hash = it.hash;
i.size = it.size;
i.name = it.name;
result.push_back(i) ;
result.push_back(i);
}
}
void RsTurtleRegExpSearchRequestItem::search(std::list<TurtleFileInfo>& result) const

73
libretroshare/src/turtle/turtleclientservice.h
View File

@ -19,23 +19,25 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
// This class is the parent class for any service that will use the turtle router to distribute its packets.
// Typical representative clients include:
//
// p3ChatService: opens tunnels to distant peers for chatting
// ftServer: searches and open tunnels to distant sources for file transfer
//
#pragma once
#include <string>
#include <stdlib.h>
#include <serialiser/rsserial.h>
#include <turtle/rsturtleitem.h>
#include "serialiser/rsserial.h"
#include "turtle/rsturtleitem.h"
#include "util/rsdebug.h"
struct RsItem;
class p3turtle ;
/** This class is the parent class for any service that will use the turtle
* router to distribute its packets.
* Typical representative clients include:
* p3ChatService: opens tunnels to distant peers for chatting
* ftServer: searches and open tunnels to distant sources for file
* transfer
*/
class RsTurtleClientService
{
public:
@ -87,30 +89,35 @@ class RsTurtleClientService
std::cerr << "!!!!!! Received Data from turtle router, but the client service is not handling it !!!!!!!!!!" << std::endl ;
}
/*!
* \brief receiveSearchRequest
* This method is called by the turtle router to notify the client of a search request in the form generic data. The returned
* result contains the serialised generic result returned by the client.
*
* The turtle router keeps the memory ownership over search_request_data
*
* \param search_request_data generic serialized search data
* \param search_request_data_len length of the serialized search data
* \param search_result_data generic serialized search result data
* \param search_result_data_len length of the serialized search result data
* \param max_allowed_hits max number of hits allowed to be sent back and forwarded
*
* \return true if the search is successful.
*/
virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,
uint32_t /*search_request_data_len*/,
unsigned char *& /*search_result_data*/,
uint32_t& /*search_result_data_len*/,
uint32_t& /* max_allows_hits */)
{
std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
return false;
}
/*!
* This method is called by the turtle router to notify the client of a
* search request in the form generic data.
* The returned result contains the serialised generic result returned by the
* client service.
* The turtle router keeps the memory ownership over search_request_data
* \param search_request_data generic serialized search data
* \param search_request_data_len length of the serialized search data
* \param search_result_data generic serialized search result data
* \param search_result_data_len length of the serialized search result data
* \param max_allowed_hits max number of hits allowed to be sent back and
* forwarded
* \return true if matching results are available, false otherwise.
*/
virtual bool receiveSearchRequest(
unsigned char *search_request_data, uint32_t search_request_data_len,
unsigned char *& search_result_data, uint32_t& search_result_data_len,
uint32_t& max_allows_hits )
{
/* Suppress unused warning this way and not commenting the param names
* so doxygen match documentation against params */
(void) search_request_data; (void) search_request_data_len;
(void) search_result_data; (void) search_result_data_len;
(void) max_allows_hits;
RS_WARN( "Received search request from turtle router, but the client "
"is not handling it!" );
return false;
}
/*!
* \brief receiveSearchResult

9
libretroshare/src/use_libretroshare.pri
View File

@ -90,6 +90,15 @@ rs_broadcast_discovery {
win32-g++|win32-clang-g++:dLibs *= wsock32
}
rs_sam3_libsam3 {
LIBSAM3_SRC_PATH=$$clean_path($${RS_SRC_PATH}/supportlibs/libsam3/)
LIBSAM3_BUILD_PATH=$$clean_path($${RS_BUILD_PATH}/supportlibs/libsam3/)
INCLUDEPATH *= $$clean_path($${LIBSAM3_SRC_PATH}/src/libsam3/)
DEPENDPATH *= $$clean_path($${LIBSAM3_BUILD_PATH})
QMAKE_LIBDIR *= $$clean_path($${LIBSAM3_BUILD_PATH})
LIBS *= -L$$clean_path($${LIBSAM3_BUILD_PATH}) -lsam3
}
static {
sLibs *= $$mLibs
} else {

2
libretroshare/src/util/dnsresolver.cc
View File

@ -41,8 +41,6 @@
const rstime_t MAX_TIME_BEFORE_RETRY = 300 ; /* seconds before retrying an ip address */
const rstime_t MAX_KEEP_DNS_ENTRY = 3600 ; /* seconds during which a DNS entry is considered valid */
static const std::string ADDR_AGENT = "Mozilla/5.0";
void *solveDNSEntries(void *p)
{
bool more_to_go = true ;

355
libretroshare/src/util/extaddrfinder.cc
View File

@ -19,9 +19,13 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
//#define EXTADDRSEARCH_DEBUG
#include "extaddrfinder.h"
#include "pqi/pqinetwork.h"
#include "rsdebug.h"
#include "util/rsstring.h"
#include "util/rsmemory.h"
@ -38,286 +42,237 @@
#include <stdio.h>
#include "util/rstime.h"
const uint32_t MAX_IP_STORE = 300; /* seconds ip address timeout */
#include <map>
//#define EXTADDRSEARCH_DEBUG
const uint32_t MAX_IP_STORE = 300; /* seconds ip address timeout */
static const std::string ADDR_AGENT = "Mozilla/5.0";
static std::string scan_ip(const std::string& text)
class ZeroInt
{
std::set<unsigned char> digits ;
digits.insert('0') ; digits.insert('3') ; digits.insert('6') ;
digits.insert('1') ; digits.insert('4') ; digits.insert('7') ;
digits.insert('2') ; digits.insert('5') ; digits.insert('8') ;
digits.insert('9') ;
public:
ZeroInt() : n(0) {}
uint32_t n ;
};
for(int i=0;i<(int)text.size();++i)
{
while(i < (int)text.size() && digits.find(text[i])==digits.end()) ++i ;
if(i>=(int)text.size())
return "" ;
unsigned int a,b,c,d ;
if(sscanf(text.c_str()+i,"%u.%u.%u.%u",&a,&b,&c,&d) != 4)
continue ;
if(a < 256 && b<256 && c<256 && d<256)
{
std::string s ;
rs_sprintf(s, "%u.%u.%u.%u", a, b, c, d) ;
return s;
}
}
return "" ;
}
static void getPage(const std::string& server_name,std::string& page)
{
page = "" ;
int sockfd,n=0; // socket descriptor
struct sockaddr_in serveur; // server's parameters
memset(&serveur.sin_zero, 0, sizeof(serveur.sin_zero));
char buf[1024];
char request[1024];
#ifdef EXTADDRSEARCH_DEBUG
std::cout << "ExtAddrFinder: connecting to " << server_name << std::endl ;
#endif
// socket creation
sockfd = unix_socket(PF_INET,SOCK_STREAM,0);
if (sockfd < 0)
{
std::cerr << "ExtAddrFinder: Failed to create socket" << std::endl;
return ;
}
serveur.sin_family = AF_INET;
// get server's ipv4 adress
in_addr in ;
if(!rsGetHostByName(server_name.c_str(),in)) /* l'hôte n'existe pas */
{
std::cerr << "ExtAddrFinder: Unknown host " << server_name << std::endl;
unix_close(sockfd);
return ;
}
serveur.sin_addr = in ;
serveur.sin_port = htons(80);
#ifdef EXTADDRSEARCH_DEBUG
printf("Connection attempt\n");
#endif
std::cerr << "ExtAddrFinder: resolved hostname " << server_name << " to " << rs_inet_ntoa(in) << std::endl;
sockaddr_storage server;
sockaddr_storage_setipv4(server, &serveur);
sockaddr_storage_setport(server, 80);
if(unix_connect(sockfd, server) == -1)
{
std::cerr << "ExtAddrFinder: Connection error to " << server_name << std::endl ;
unix_close(sockfd);
return ;
}
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: Connection established to " << server_name << std::endl ;
#endif
// envoi
if(snprintf( request,
1024,
"GET / HTTP/1.0\r\n"
"Host: %s:%d\r\n"
"Connection: Close\r\n"
"\r\n",
server_name.c_str(), 80) > 1020)
{
std::cerr << "ExtAddrFinder: buffer overrun. The server name \"" << server_name << "\" is too long. This is quite unexpected." << std::endl;
unix_close(sockfd);
return ;
}
if(send(sockfd,request,strlen(request),0)== -1)
{
std::cerr << "ExtAddrFinder: Could not send request to " << server_name << std::endl ;
unix_close(sockfd);
return ;
}
// recéption
while((n = recv(sockfd, buf, sizeof buf - 1, 0)) > 0)
{
buf[n] = '\0';
page += std::string(buf,n) ;
}
// fermeture de la socket
unix_close(sockfd);
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: Got full page from " << server_name << std::endl ;
#endif
}
void* doExtAddrSearch(void *p)
void ExtAddrFinder::run()
{
std::vector<std::string> res ;
ExtAddrFinder *af = (ExtAddrFinder*)p ;
for(std::list<std::string>::const_iterator it(af->_ip_servers.begin());it!=af->_ip_servers.end();++it)
for(auto& it : _ip_servers)
{
std::string page ;
getPage(*it,page) ;
std::string ip = scan_ip(page) ;
std::string ip = "";
rsGetHostByNameSpecDNS(it,"myip.opendns.com",ip,2);
if(ip != "")
res.push_back(ip) ;
#ifdef EXTADDRSEARCH_DEBUG
std::cout << "ip found through " << *it << ": \"" << ip << "\"" << std::endl ;
RS_DBG("ip found through DNS ", it, ": \"", ip, "\"");
#endif
}
if(res.empty())
{
// thread safe copy results.
//
{
RsStackMutex mtx(af->mAddrMtx) ;
af->mFound = false ;
af->mFoundTS = time(NULL) ;
af->mSearching = false ;
}
return NULL ;
reset();
return ;
}
sort(res.begin(),res.end()) ; // eliminates outliers.
std::map<sockaddr_storage,ZeroInt> addrV4_votes;
std::map<sockaddr_storage,ZeroInt> addrV6_votes;
std::string all_addrV4_Found;
std::string all_addrV6_Found;
if(!sockaddr_storage_ipv4_aton(af->mAddr, res[res.size()/2].c_str()))
for(auto curRes : res)
{
std::cerr << "ExtAddrFinder: Could not convert " << res[res.size()/2] << " into an address." << std::endl ;
sockaddr_storage addr;
sockaddr_storage_clear(addr);
bool validIP = sockaddr_storage_inet_pton(addr, curRes)
&& sockaddr_storage_isValidNet(addr);
bool isIPv4 = sockaddr_storage_ipv6_to_ipv4(addr);
if( validIP && isIPv4 )
{
RsStackMutex mtx(af->mAddrMtx) ;
af->mFound = false ;
af->mFoundTS = time(NULL) ;
af->mSearching = false ;
addr.ss_family = AF_INET;
addrV4_votes[addr].n++ ;
all_addrV4_Found += sockaddr_storage_tostring(addr) + "\n";
}
return NULL ;
else if( validIP && !isIPv4)
{
addr.ss_family = AF_INET6;
addrV6_votes[addr].n++ ;
all_addrV6_Found += sockaddr_storage_tostring(addr) + "\n";
}
else
RS_ERR("Invalid addresse reported: ", curRes) ;
}
if( (0 == addrV4_votes.size()) && (0 == addrV6_votes.size()) )
{
RS_ERR("Could not find any external address.");
reset();
return ;
}
if( 1 < addrV4_votes.size() )
RS_ERR("Multiple external IPv4 addresses reported: "
, all_addrV4_Found ) ;
if( 1 < addrV6_votes.size() )
RS_ERR("Multiple external IPv6 addresses reported: "
, all_addrV6_Found ) ;
{
RsStackMutex mtx(af->mAddrMtx) ;
af->mFound = true ;
af->mFoundTS = time(NULL) ;
af->mSearching = false ;
RS_STACK_MUTEX(mAddrMtx);
mSearching = false ;
mFoundTS = time(NULL) ;
// Only save more reported address if not only once.
uint32_t admax = 0 ;
sockaddr_storage_clear(mAddrV4);
for (auto it : addrV4_votes)
if (admax < it.second.n)
{
mAddrV4 = it.first ;
mFoundV4 = true ;
admax = it.second.n ;
}
admax = 0 ;
sockaddr_storage_clear(mAddrV6);
for (auto it : addrV6_votes)
if (admax < it.second.n)
{
mAddrV6 = it.first ;
mFoundV6 = true ;
admax = it.second.n ;
}
}
return NULL ;
return ;
}
void ExtAddrFinder::start_request()
{
void *data = (void *)this;
pthread_t tid ;
if(! pthread_create(&tid, 0, &doExtAddrSearch, data))
pthread_detach(tid); /* so memory is reclaimed in linux */
else
std::cerr << "(EE) Could not start ExtAddrFinder thread." << std::endl;
if (!isRunning())
start("ExtAddrFinder");
}
bool ExtAddrFinder::hasValidIP(struct sockaddr_storage &addr)
bool ExtAddrFinder::hasValidIPV4(struct sockaddr_storage &addr)
{
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: Getting ip." << std::endl ;
RS_DBG("Getting ip.");
#endif
{
RsStackMutex mut(mAddrMtx) ;
if(mFound)
RS_STACK_MUTEX(mAddrMtx) ;
if(mFoundV4)
{
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: Has stored ip: responding with this ip." << std::endl ;
RS_DBG("Has stored ip responding with this ip:", sockaddr_storage_iptostring(mAddrV4)) ;
#endif
sockaddr_storage_copyip(addr,mAddr); // just copy the IP so we dont erase the port.
sockaddr_storage_copyip(addr,mAddrV4); // just copy the IP so we dont erase the port.
}
}
rstime_t delta;
testTimeOut();
RS_STACK_MUTEX(mAddrMtx) ;
return mFoundV4;
}
bool ExtAddrFinder::hasValidIPV6(struct sockaddr_storage &addr)
{
#ifdef EXTADDRSEARCH_DEBUG
RS_DBG("Getting ip.");
#endif
{
RsStackMutex mut(mAddrMtx) ;
//timeout the current ip
delta = time(NULL) - mFoundTS;
RS_STACK_MUTEX(mAddrMtx) ;
if(mFoundV6)
{
#ifdef EXTADDRSEARCH_DEBUG
RS_DBG("Has stored ip responding with this ip:", sockaddr_storage_iptostring(mAddrV6)) ;
#endif
sockaddr_storage_copyip(addr,mAddrV6); // just copy the IP so we dont erase the port.
}
}
if((uint32_t)delta > MAX_IP_STORE) {//launch a research
testTimeOut();
RS_STACK_MUTEX(mAddrMtx) ;
return mFoundV6;
}
void ExtAddrFinder::testTimeOut()
{
bool timeOut;
{
RS_STACK_MUTEX(mAddrMtx) ;
//timeout the current ip
timeOut = (mFoundTS + MAX_IP_STORE < time(NULL));
}
if(timeOut || mFirstTime) {//launch a research
if( mAddrMtx.trylock())
{
if(!mSearching)
{
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: No stored ip: Initiating new search." << std::endl ;
RS_DBG("No stored ip: Initiating new search.");
#endif
mSearching = true ;
start_request() ;
}
#ifdef EXTADDRSEARCH_DEBUG
else
std::cerr << "ExtAddrFinder: Already searching." << std::endl ;
RS_DBG("Already searching.");
#endif
mFirstTime = false;
mAddrMtx.unlock();
}
#ifdef EXTADDRSEARCH_DEBUG
else
std::cerr << "ExtAddrFinder: (Note) Could not acquire lock. Busy." << std::endl ;
RS_DBG("(Note) Could not acquire lock. Busy.");
#endif
}
RsStackMutex mut(mAddrMtx) ;
return mFound ;
}
void ExtAddrFinder::reset()
void ExtAddrFinder::reset(bool firstTime /*=false*/)
{
RsStackMutex mut(mAddrMtx) ;
#ifdef EXTADDRSEARCH_DEBUG
RS_DBG("firstTime=", firstTime);
#endif
RS_STACK_MUTEX(mAddrMtx) ;
mFound = false ;
mSearching = false ;
mFoundTS = time(NULL) - MAX_IP_STORE;
mFoundV4 = false ;
mFoundV6 = false ;
mFirstTime = firstTime;
mFoundTS = time(nullptr);
sockaddr_storage_clear(mAddrV4);
sockaddr_storage_clear(mAddrV6);
}
ExtAddrFinder::~ExtAddrFinder()
{
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: Deleting ExtAddrFinder." << std::endl ;
RS_DBG("Deleting ExtAddrFinder.");
#endif
}
ExtAddrFinder::ExtAddrFinder() : mAddrMtx("ExtAddrFinder")
{
#ifdef EXTADDRSEARCH_DEBUG
std::cerr << "ExtAddrFinder: Creating new ExtAddrFinder." << std::endl ;
RS_DBG("Creating new ExtAddrFinder.");
#endif
RsStackMutex mut(mAddrMtx) ;
reset( true );
mFound = false;
mSearching = false;
mFoundTS = time(NULL) - MAX_IP_STORE;
sockaddr_storage_clear(mAddr);
_ip_servers.push_back(std::string( "checkip.dyndns.org" )) ;
_ip_servers.push_back(std::string( "www.myip.dk" )) ;
_ip_servers.push_back(std::string( "showip.net" )) ;
_ip_servers.push_back(std::string( "www.displaymyip.com")) ;
//https://unix.stackexchange.com/questions/22615/how-can-i-get-my-external-ip-address-in-a-shell-script
//Enter direct ip so local DNS cannot change it.
//DNS servers must recognize "myip.opendns.com"
_ip_servers.push_back(std::string( "208.67.222.222" )) ;//resolver1.opendns.com
_ip_servers.push_back(std::string( "208.67.220.220" )) ;//resolver2.opendns.com
_ip_servers.push_back(std::string( "208.67.222.220" )) ;//resolver3.opendns.com
_ip_servers.push_back(std::string( "208.67.220.222" )) ;//resolver4.opendns.com
_ip_servers.push_back(std::string( "2620:119:35::35" )) ;//resolver1.opendns.com
_ip_servers.push_back(std::string( "2620:119:53::53" )) ;//resolver2.opendns.com
}

25
libretroshare/src/util/extaddrfinder.h
View File

@ -30,26 +30,31 @@
struct sockaddr ;
class ExtAddrFinder
class ExtAddrFinder: public RsThread
{
public:
ExtAddrFinder() ;
~ExtAddrFinder() ;
bool hasValidIP(struct sockaddr_storage &addr) ;
bool hasValidIPV4(struct sockaddr_storage &addr) ;
bool hasValidIPV6(struct sockaddr_storage &addr) ;
void getIPServersList(std::list<std::string>& ip_servers) { ip_servers = _ip_servers ; }
void start_request() ;
void reset() ;
void reset(bool firstTime = false) ;
private:
friend void* doExtAddrSearch(void *p) ;
virtual void run();
void testTimeOut();
RsMutex mAddrMtx ;
rstime_t mFoundTS;
struct sockaddr_storage mAddr;
bool mFound ;
bool mSearching ;
std::list<std::string> _ip_servers ;
RsMutex mAddrMtx;
bool mSearching;
bool mFoundV4;
bool mFoundV6;
bool mFirstTime;
rstime_t mFoundTS;
struct sockaddr_storage mAddrV4;
struct sockaddr_storage mAddrV6;
std::list<std::string> _ip_servers;
};

7
libretroshare/src/util/folderiterator.cc
View File

@ -38,8 +38,11 @@
namespace librs { namespace util {
FolderIterator::FolderIterator(const std::string& folderName, bool allow_symlinks, bool allow_files_from_the_future)
: mFolderName(folderName),mAllowSymLinks(allow_symlinks),mAllowFilesFromTheFuture(allow_files_from_the_future)
FolderIterator::FolderIterator(
const std::string& folderName, bool allow_symlinks,
bool allow_files_from_the_future ):
mFolderName(folderName), mAllowSymLinks(allow_symlinks),
mAllowFilesFromTheFuture(allow_files_from_the_future)
{
is_open = false ;
validity = false ;

6
libretroshare/src/util/folderiterator.h
View File

@ -44,8 +44,10 @@ namespace librs { namespace util {
class FolderIterator
{
public:
FolderIterator(const std::string& folderName,bool allow_symlinks,bool allow_files_from_the_future = true);
~FolderIterator();
FolderIterator(
const std::string& folderName, bool allow_symlinks,
bool allow_files_from_the_future = true );
~FolderIterator();
enum { TYPE_UNKNOWN = 0x00,
TYPE_FILE = 0x01,

122
libretroshare/src/util/i2pcommon.cpp
View File

@ -50,8 +50,10 @@ std::string publicKeyFromPrivate(std::string const &priv)
* https://geti2p.net/spec/common-structures#keysandcert
* https://geti2p.net/spec/common-structures#certificate
*/
if (priv.length() < 884) // base64 ( = 663 bytes = KeyCert + priv Keys)
if (priv.length() < privKeyMinLenth_b64) {
RS_WARN("key to short!");
return std::string();
}
// creat a copy to work on, need to convert it to standard base64
auto priv_copy(priv);
@ -70,13 +72,14 @@ std::string publicKeyFromPrivate(std::string const &priv)
uint8_t certType = 0;
uint16_t len = 0;
uint16_t signingKeyType = 0;
uint16_t cryptKey = 0;
uint16_t cryptKeyType = 0;
// only used for easy break
do {
try {
// jump to certificate
p += publicKeyLen;
// try to read type and length
certType = *p++;
len = readTwoBytesBE(p);
@ -87,7 +90,7 @@ std::string publicKeyFromPrivate(std::string const &priv)
/*
* CertType.Null
* type null is followed by 0x00 0x00 <END>
* so has to be 0!
* so len has to be 0!
*/
RS_DBG("cert is CertType.Null");
publicKeyLen += 3; // add 0x00 0x00 0x00
@ -119,7 +122,7 @@ std::string publicKeyFromPrivate(std::string const &priv)
// likely 7
signingKeyType = readTwoBytesBE(p);
RS_DBG("signing pubkey type ", certType);
RS_DBG("signing pubkey type ", signingKeyType);
if (signingKeyType >= 3 && signingKeyType <= 6) {
RS_DBG("signing pubkey type ", certType, " has oversize");
// calculate oversize
@ -137,18 +140,18 @@ std::string publicKeyFromPrivate(std::string const &priv)
return std::string();
}
publicKeyLen += values.first - 128; // 128 = default DSA key length = the space than can be used before the key must be splitted
publicKeyLen += values.first - 128; // 128 = default DSA key length = the space that can be used before the key must be splitted
}
// Crypto Public Key
// likely 0
cryptKey = readTwoBytesBE(p);
RS_DBG("crypto pubkey type ", cryptKey);
cryptKeyType = readTwoBytesBE(p);
RS_DBG("crypto pubkey type ", cryptKeyType);
// info: these are all smaller than the default 256 bytes, so no oversize calculation is needed
break;
} catch (const std::out_of_range &e) {
RS_DBG("hit exception! ", e.what());
RS_DBG("hit an exception! ", e.what());
return std::string();
}
} while(false);
@ -160,4 +163,107 @@ std::string publicKeyFromPrivate(std::string const &priv)
return pub;
}
bool getKeyTypes(const std::string &key, std::string &signingKey, std::string &cryptoKey)
{
if (key.length() < pubKeyMinLenth_b64) {
RS_WARN("key to short!");
return false;
}
// creat a copy to work on, need to convert it to standard base64
auto key_copy(key);
std::replace(key_copy.begin(), key_copy.end(), '~', '/');
// replacing the - with a + is not necessary, as RsBase64 can handle base64url encoding, too
// std::replace(copy.begin(), copy.end(), '-', '+');
// get raw data
std::vector<uint8_t> data;
RsBase64::decode(key_copy, data);
auto p = data.cbegin();
constexpr size_t publicKeyLen = 256 + 128; // default length (bytes)
uint8_t certType = 0;
uint16_t signingKeyType = 0;
uint16_t cryptKeyType = 0;
// try to read types
try {
// jump to certificate
p += publicKeyLen;
// try to read type and skip length
certType = *p++;
p += 2;
// only 0 and 5 are used / valid at this point
// check for == 0
if (certType == static_cast<typename std::underlying_type<CertType>::type>(CertType::Null)) {
RS_DBG("cert is CertType.Null");
signingKey = "DSA_SHA1";
cryptoKey = "ElGamal";
return true;
}
// check for != 5
if (certType != static_cast<typename std::underlying_type<CertType>::type>(CertType::Key)) {
// unsupported
RS_DBG("cert type ", certType, " is unsupported");
return false;
}
RS_DBG("cert is CertType.Key");
// Signing Public Key
// likely 7
signingKeyType = readTwoBytesBE(p);
RS_DBG("signing pubkey type ", signingKeyType);
// Crypto Public Key
// likely 0
cryptKeyType = readTwoBytesBE(p);
RS_DBG("crypto pubkey type ", cryptKeyType);
} catch (const std::out_of_range &e) {
RS_DBG("hit an exception! ", e.what());
return false;
}
// now convert to string (this would be easier with c++17)
#define HELPER(a, b, c) \
case static_cast<typename std::underlying_type<a>::type>(a::c): \
b = #c; \
break;
switch (signingKeyType) {
HELPER(SigningKeyType, signingKey, DSA_SHA1)
HELPER(SigningKeyType, signingKey, ECDSA_SHA256_P256)
HELPER(SigningKeyType, signingKey, ECDSA_SHA384_P384)
HELPER(SigningKeyType, signingKey, ECDSA_SHA512_P521)
HELPER(SigningKeyType, signingKey, RSA_SHA256_2048)
HELPER(SigningKeyType, signingKey, RSA_SHA384_3072)
HELPER(SigningKeyType, signingKey, RSA_SHA512_4096)
HELPER(SigningKeyType, signingKey, EdDSA_SHA512_Ed25519)
HELPER(SigningKeyType, signingKey, EdDSA_SHA512_Ed25519ph)
HELPER(SigningKeyType, signingKey, RedDSA_SHA512_Ed25519)
default:
RsWarn("unkown signing key type:", signingKeyType);
return false;
}
switch (cryptKeyType) {
HELPER(CryptoKeyType, cryptoKey, ElGamal)
HELPER(CryptoKeyType, cryptoKey, P256)
HELPER(CryptoKeyType, cryptoKey, P384)
HELPER(CryptoKeyType, cryptoKey, P521)
HELPER(CryptoKeyType, cryptoKey, X25519)
default:
RsWarn("unkown crypto key type:", cryptKeyType);
return false;
}
#undef HELPER
return true;
}
} // namespace i2p

42
libretroshare/src/util/i2pcommon.h
View File

@ -186,6 +186,39 @@ static const std::array<std::pair<uint16_t, uint16_t>, 12> signingKeyLengths {
/*SigningKeyType::RedDSA_SHA512_Ed25519 */ std::make_pair<uint16_t, uint16_t>( 32, 32),
};
/*
* Key length infos:
*
* BOB private key
* len b64: 884
* len pln: 663
*
* BOB public key / destination
* len b64: 516
* len pln: 387
*
* SAMv3 private key
* len b64: 908
* len pln: 679
*
* SAMv3 public key
* len b64: 516
* len pln: 387
*
* Example:
* in bytes, public key only
* 384 (Key) + 3 (Null certificate) = 387 bytes
* 384 (Key) + 7 (key certificate) = 391 bytes
*
* in bytes public + private key
* 384 (Key) + 3 (Null certificate) + 256 (ElGamal) + 20 (DSA_SHA1) = 663 bytes
* 384 (Key) + 7 (key certificate) + 256 (ElGamal) + 32 (EdDSA_SHA512_Ed25519) = 679 bytes
*/
constexpr size_t pubKeyMinLenth_b64 = 516;
constexpr size_t pubKeyMinLenth_bin = 387;
constexpr size_t privKeyMinLenth_b64 = 884;
constexpr size_t privKeyMinLenth_bin = 663;
/**
* @brief makeOption Creates the string "lhs=rhs" used by BOB and SAM. Converts rhs
* @param lhs option to set
@ -208,6 +241,15 @@ std::string keyToBase32Addr(const std::string &key);
*/
std::string publicKeyFromPrivate(const std::string &priv);
/**
* @brief getKeyTypes returns the name of the utilized algorithms used by the key
* @param key public key
* @param signingKey name of the signing key, e.g. DSA_SHA1
* @param cryptoKey name of the crpyto key, e.g. ElGamal
* @return true on success, false otherwise
*/
bool getKeyTypes(const std::string &key, std::string &signingKey, std::string &cryptoKey);
} // namespace i2p
#endif // I2PCOMMON_H

60
libretroshare/src/util/rsdebug.cc
View File

@ -3,9 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2004-2008 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2020-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -22,6 +22,8 @@
* *
*******************************************************************************/
#include <iomanip>
#include "util/rsdebug.h"
std::ostream &operator<<(std::ostream& out, const std::error_condition& err)
@ -36,6 +38,54 @@ std::string rsErrorNotInCategory(int errNum, const std::string& categoryName)
" not available in category: " + categoryName;
}
std::error_condition rs_errno_to_condition(int errno_code)
{ return std::make_error_condition(static_cast<std::errc>(errno_code)); }
std::ostream& hex_dump(std::ostream& os, const void *buffer,
std::size_t bufsize, bool showPrintableChars /*= true*/)
{
if (buffer == nullptr) {
return os;
}
auto oldFormat = os.flags();
auto oldFillChar = os.fill();
constexpr std::size_t maxline{8};
// create a place to store text version of string
char renderString[maxline+1];
char *rsptr{renderString};
// convenience cast
const unsigned char *buf{reinterpret_cast<const unsigned char *>(buffer)};
for (std::size_t linecount=maxline; bufsize; --bufsize, ++buf) {
os << std::setw(2) << std::setfill('0') << std::hex
<< static_cast<unsigned>(*buf) << ' ';
*rsptr++ = std::isprint(*buf) ? *buf : '.';
if (--linecount == 0) {
*rsptr++ = '\0'; // terminate string
if (showPrintableChars) {
os << " | " << renderString;
}
os << '\n';
rsptr = renderString;
linecount = std::min(maxline, bufsize);
}
}
// emit newline if we haven't already
if (rsptr != renderString) {
if (showPrintableChars) {
for (*rsptr++ = '\0'; rsptr != &renderString[maxline+1]; ++rsptr) {
os << " ";
}
os << " | " << renderString;
}
os << '\n';
}
os.fill(oldFillChar);
os.flags(oldFormat);
return os;
}
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
@ -208,3 +258,7 @@ void rslog(const RsLog::logLvl lvl, RsLog::logInfo *info, const std::string &msg
lineCount++;
}
}
/// All the lines before are DEPRECATED!!
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////

62
libretroshare/src/util/rsdebug.h
View File

@ -2,8 +2,8 @@
* RetroShare debugging utilities *
* *
* Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2019-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -65,6 +65,12 @@ std::ostream &operator<<(std::ostream& out, const std::error_condition& err);
* the message around */
std::string rsErrorNotInCategory(int errNum, const std::string& categoryName);
/** Convert C errno codes to modern C++11 std::error_condition, this is quite
* useful to use toghether with C functions used around the code like `malloc`,
* `socket` etc to let errors bubble up comprensibly to upper layers C++11 code
*/
std::error_condition rs_errno_to_condition(int errno_code);
template <RsLoggerCategories CATEGORY>
struct t_RsLogger : std::ostringstream
@ -204,6 +210,53 @@ struct RsNoDbg
};
//From https://codereview.stackexchange.com/a/165162
/**
* @brief hex_dump: Send Hexadecimal Dump to stream
* @param os: Output Stream
* @param buffer: Buffer to send
* @param bufsize: Buffer's size
* @param showPrintableChars: If must send printable Char too
* @return
* basic string:
* 61 62 63 64 65 66 31 32 | abcdef12
* 33 34 35 36 00 7a 79 78 | 3456.zyx
* 77 76 75 39 38 37 36 35 | wvu98765
* 34 45 64 77 61 72 64 00 | 4Edward.
*
* wide string:
* 41 00 00 00 20 00 00 00 | A... ...
* 77 00 00 00 69 00 00 00 | w...i...
* 64 00 00 00 65 00 00 00 | d...e...
* 20 00 00 00 73 00 00 00 | ...s...
* 74 00 00 00 72 00 00 00 | t...r...
* 69 00 00 00 6e 00 00 00 | i...n...
* 67 00 00 00 2e 00 00 00 | g.......
*
* a double
* 49 92 24 49 92 24 09 40 | I.$I.$.@
*/
std::ostream& hex_dump(std::ostream& os, const void *buffer,
std::size_t bufsize, bool showPrintableChars = true);
/**
* @brief The hexDump struct
* Enable to print dump calling like that:
* const char test[] = "abcdef123456\0zyxwvu987654Edward";
* RsDbg()<<hexDump(test, sizeof(test))<<std::endl;
*/
struct hexDump {
const void *buffer;
std::size_t bufsize;
hexDump(const void *buf, std::size_t bufsz) : buffer{buf}, bufsize{bufsz} {}
friend std::ostream &operator<<(std::ostream &out, const hexDump &hd) {
return hex_dump(out, hd.buffer, hd.bufsize, true);
}
};
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
@ -312,3 +365,8 @@ void rslog(const RsLog::logLvl lvl, RsLog::logInfo *info, const std::string &msg
#define PQL_DEBUG_ALERT RSL_DEBUG_ALERT
#define PQL_DEBUG_BASIC RSL_DEBUG_BASIC
#define PQL_DEBUG_ALL RSL_DEBUG_ALL
/// All the lines before are DEPRECATED!!
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////

30
libretroshare/src/util/rsdir.cc
View File

@ -458,6 +458,36 @@ bool RsDirUtil::checkFile(const std::string& filename,uint64_t& file_size,bool d
return true;
}
rstime_t RsDirUtil::lastWriteTime(
const std::string& path,
std::error_condition& errc )
{
if(!fileExists(path))
{
errc = std::errc::no_such_file_or_directory;
return 0;
}
#ifdef WINDOWS_SYS
struct _stati64 buf;
std::wstring wPath;
librs::util::ConvertUtf8ToUtf16(path, wPath);
if ( 0 == _wstati64(wPath.c_str(), &buf))
#else
struct stat64 buf;
if ( 0 == stat64(path.c_str(), &buf))
#endif
{
/* errc output param is guaranted to be meaningful only if an error
* happens so is not strictly necessary but we clean it anyway just
* in case */
errc = std::error_condition();
return buf.st_mtime;
}
errc = std::error_condition(errno, std::generic_category());
return 0;
}
bool RsDirUtil::checkDirectory(const std::string& dir)
{

30
libretroshare/src/util/rsdir.h
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2004-2007 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2020-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -27,11 +27,13 @@
#include <string>
#include <list>
#include <set>
#include <stdint.h>
#include <cstdint>
#include <system_error>
class RsThread;
#include <retroshare/rstypes.h>
#include "retroshare/rstypes.h"
#include "util/rsmemory.h"
#ifndef WINDOWS_SYS
typedef int rs_lock_handle_t;
@ -79,10 +81,11 @@ const char *scanf_string_for_uint(int bytes) ;
int breakupDirList(const std::string& path, std::list<std::string> &subdirs);
// Splits the path into parent directory and file. File can be empty if full_path is a dir ending with '/'
// if full_path does not contain a directory, then dir will be "." and file will be full_path.
bool splitDirFromFile(const std::string& full_path,std::string& dir, std::string& file);
/** Splits the path into parent directory and file. File can be empty if
* full_path is a dir ending with '/' if full_path does not contain a directory,
* then dir will be "." and file will be full_path */
bool splitDirFromFile( const std::string& full_path,
std::string& dir, std::string& file );
bool copyFile(const std::string& source,const std::string& dest);
@ -92,6 +95,17 @@ bool moveFile(const std::string& source, const std::string& dest);
bool removeFile(const std::string& file);
bool fileExists(const std::string& file);
bool checkFile(const std::string& filename,uint64_t& file_size,bool disallow_empty_file = false);
/**
* @brief Retrieve file last modification time
* @param path path of the file
* @param errc optional storage for error details
* @return 0 on error, file modification time represented as unix epoch otherwise.
*/
rstime_t lastWriteTime(
const std::string& path,
std::error_condition& errc = RS_DEFAULT_STORAGE_PARAM(std::error_condition) );
bool checkDirectory(const std::string& dir);
bool checkCreateDirectory(const std::string& dir);

361
libretroshare/src/util/rsdnsutils.cc Normal file
View File

@ -0,0 +1,361 @@
/*******************************************************************************
* libretroshare/src/util: rsdnsutils.cc *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2021 Phenom <retrosharephenom@gmail.com> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
//#define DEBUG_SPEC_DNS 1
#include "util/rsnet.h"
#include "util/rsdebug.h"
#include "util/rsthreads.h"
#include "util/rsstring.h"
#ifdef WINDOWS_SYS
#else
#include <netdb.h>
#endif
//https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-2
constexpr uint16_t DNSC_IN = 1; //Internet (IN)
//https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4
constexpr uint16_t DNST_A = 1; //Ipv4 address
constexpr uint16_t DNST_AAAA =28; //Ipv6 address
///Need to pack as we use sizeof them. (avoid padding)
#pragma pack(1)
//DNS header structure
struct DNS_HEADER
{
unsigned short id; // identification number
//Header flags https://www.bind9.net/dns-header-flags
unsigned char rd :1; //bit 07 Recursion Desired, indicates if the client means a recursive query
unsigned char tc :1; //bit 06 TrunCation, indicates that this message was truncated due to excessive length
unsigned char aa :1; //bit 05 Authoritative Answer, in a response, indicates if the DNS server is authoritative for the queried hostname
unsigned char opcode :4;//bit 01-04 The type can be QUERY (standard query, 0), IQUERY (inverse query, 1), or STATUS (server status request, 2)
unsigned char qr :1; //bit 00 Indicates if the message is a query (0) or a reply (1)
unsigned char rcode :4; //bit 12-15 Response Code can be NOERROR (0), FORMERR (1, Format error), SERVFAIL (2), NXDOMAIN (3, Nonexistent domain), etc.
unsigned char cd :1; //bit 11 Checking Disabled [RFC 4035][RFC 6840][RFC Errata 4927] used by DNSSEC
unsigned char ad :1; //bit 10 Authentic Data [RFC 4035][RFC 6840][RFC Errata 4924] used by DNSSEC
unsigned char z :1; //bit 09 Zero, reserved for future use
unsigned char ra :1; //bit 08 Recursion Available [RFC 1035] in a response, indicates if the replying DNS server supports recursion
unsigned short q_count; // number of question entries
unsigned short ans_count; // number of answer entries
unsigned short auth_count; // number of authority resource records entries
unsigned short add_count; // number of additional resource record entries
};
//// OpCode text https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-5
//static const char *opcodetext[] = { "QUERY", "IQUERY", "STATUS",
// "RESERVED3", "NOTIFY", "UPDATE",
// "STATEFUL", "RESERVED7", "RESERVED8",
// "RESERVED9", "RESERVED10", "RESERVED11",
// "RESERVED12", "RESERVED13", "RESERVED14",
// "RESERVED15" };
//// RCode text https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-6
//static const char *rcodetext[] = { "NOERROR", "FORMERR", "SERVFAIL",
// "NXDOMAIN", "NOTIMP", "REFUSED",
// "YXDOMAIN", "YXRRSET", "NXRRSET",
// "NOTAUTH", "NOTZONE", "DSOTYPENI",
// "RESERVED12", "RESERVED13", "RESERVED14",
// "RESERVED15", "BADVERS", "BADKEY",
// "BADTIME", "BADMODE", "BADNAME",
// "BADALG", "BADTRUNC", "BADCOOKIE"};
//Constant sized fields of query structure
//https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml
struct QUESTION
{
unsigned short qtype; //https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4
unsigned short qclass; //https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-2
};
//Constant sized fields of the resource record structure
struct RR_DATA
{
unsigned short rtype; //https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-4
unsigned short rclass; //https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-2
unsigned int rttl; //Time To Live is the number of seconds left before the information expires. (32bits integer, so maximum 140 years)
unsigned short data_len;//Lenght of following data
};
#pragma pack()
bool rsGetHostByNameSpecDNS(const std::string& servername, const std::string& hostname, std::string& returned_addr, int timeout_s /*= -1*/)
{
#ifdef DEBUG_SPEC_DNS
RS_DBG("servername=", servername, " hostname=", hostname);
#endif
if (strlen(servername.c_str()) > 256)
{
RS_ERR("servername is too long > 256 chars: ", servername);
return false;
}
if (strlen(hostname.c_str()) > 256)
{
RS_ERR("hostname is too long > 256 chars: ", hostname);
return false;
}
sockaddr_storage serverAddr;
bool validServer = sockaddr_storage_inet_pton(serverAddr, servername)
&& sockaddr_storage_isValidNet(serverAddr);
bool isIPV4 = validServer && sockaddr_storage_ipv6_to_ipv4(serverAddr);
if (!validServer)
{
in_addr in ;
if (!rsGetHostByName(servername, in))
{
RS_ERR("servername is on an unknow format: ", servername);
return false;
}
validServer = sockaddr_storage_inet_pton(serverAddr, rs_inet_ntoa(in))
&& sockaddr_storage_isValidNet(serverAddr);
isIPV4 = validServer && sockaddr_storage_ipv6_to_ipv4(serverAddr);
if (!validServer)
{
RS_ERR("rsGetHostByName return bad answer: ", rs_inet_ntoa(in));
return false;
}
}
sockaddr_storage_setport( serverAddr, 53);
//Set the DNS structure to standard queries
unsigned char buf[65536];
struct DNS_HEADER* dns = (struct DNS_HEADER *)&buf;
dns->id = static_cast<unsigned short>(htons(getpid())); //Transaction Id
//dns flags = 0x0100 Standard Query
dns->qr = 0; //Query/Response: Message is a query
dns->opcode = 0; //OpCode: Standard query
dns->aa = 0; //Authoritative: Server is not an authority for domain
dns->tc = 0; //TrunCated: Message is not truncated
dns->rd = 1; //Recursion Desired: Do query recursively
dns->ra = 0; //Recursion Available: Server cannot do recursive queries
dns->z = 0; //Z: reserved
dns->ad = 0; //Authentic Data: Answer/authority portion was not authenticated by the server
dns->cd = 0; //Checking Disabled: Unacceptable
dns->rcode = 0; //Response Code: No error
dns->q_count = htons(1); //1 Question
dns->ans_count = 0; //0 Answer
dns->auth_count = 0; //0 Authority RRs
dns->add_count = 0; //0 Additional RRs
size_t curSendSize = sizeof(struct DNS_HEADER);
//Point to the query server name portion
unsigned char* qname =static_cast<unsigned char*>(&buf[curSendSize]);
//First byte is Label Type: https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-10
qname[0] = 0x04; //One Label with Normal label lower 6 bits is the length of the label
memcpy(&qname[1],hostname.c_str(),strlen(hostname.c_str()));
size_t qnameSize = strlen((const char*)qname);
// Format Hostname like www.google.com to 3www6google3com
{
size_t last = qnameSize;
for(size_t i = qnameSize-1 ; i > 0 ; i--)
if(qname[i]=='.')
{
qname[i]=last-i-1;
last = i;
}
}
curSendSize += qnameSize +1; //With \0 terminator
//Point to the query constant portion
struct QUESTION* qinfo =(struct QUESTION*)&buf[curSendSize];
qinfo->qtype = htons(isIPV4 ? DNST_A : DNST_AAAA); //Type: A / AAAA(Host Address)
qinfo->qclass = htons(DNSC_IN); //Class: IN
curSendSize += sizeof(struct QUESTION);
#ifdef DEBUG_SPEC_DNS
RS_DBG("Sending Packet:\n", hexDump(buf, curSendSize));
#endif
int s = socket(serverAddr.ss_family , SOCK_DGRAM , IPPROTO_UDP); //UDP packet for DNS queries
if(s<0)
{
RS_ERR("Could not open socket.");
return false;
}
if (timeout_s > -1)
rs_setSockTimeout(s, true, timeout_s);
ssize_t send_size = sendto( s, (char*)buf, curSendSize, 0
, (struct sockaddr*)&serverAddr
, isIPV4 ? sizeof(sockaddr_in)
: sizeof(sockaddr_in6)
);
if( send_size < 0)
{
RS_ERR("Send Failed with size = ", send_size);
close(s);
return false;
}
#ifdef DEBUG_SPEC_DNS
RS_DBG("Waiting answer...");
#endif
//****************************************************************************************//
//--- Receive the answer ---//
//****************************************************************************************//
socklen_t sa_size = static_cast<socklen_t>(isIPV4 ? sizeof(sockaddr_in)
: sizeof(sockaddr_in6)
);
ssize_t rec_size=recvfrom( s,(char*)buf , 65536 , 0
, (struct sockaddr*)&serverAddr
, &sa_size
);
close(s); // No more need of this socket, close it.
if(rec_size <= 0)
{
RS_ERR("Receive Failed");
return false;
}
#ifdef DEBUG_SPEC_DNS
RS_DBG("Received:\n", hexDump(buf, rec_size));
#endif
if (rec_size< static_cast<ssize_t>(sizeof(struct DNS_HEADER)) )
{
RS_ERR("Request received too small to get DNSHeader.");
return false;
}
#ifdef DEBUG_SPEC_DNS
//Point to the header portion
dns = (struct DNS_HEADER*) buf;
RS_DBG("The response contains :\n"
,ntohs(dns->q_count) , " Questions.\n"
,ntohs(dns->ans_count) , " Answers.\n"
,ntohs(dns->auth_count) , " Authoritative Servers.\n"
,ntohs(dns->add_count) , " Additional records.");
#endif
size_t curRecSize = sizeof(struct DNS_HEADER);
if (rec_size< static_cast<ssize_t>(curRecSize + 1 + sizeof(struct QUESTION)) )
{
RS_ERR("Request received too small to get Question return.");
return false;
}
//Point to the query portion
unsigned char* qnameRecv =static_cast<unsigned char*>(&buf[curRecSize]);
if (memcmp(qname,qnameRecv,qnameSize + 1 + sizeof(struct QUESTION)) )
{
RS_ERR("Request received different from that sent.");
return false;
}
curRecSize += qnameSize + 1 + sizeof(struct QUESTION);
if (rec_size< static_cast<ssize_t>(curRecSize + 2) )
{
RS_ERR("Request received too small to get Answer return.");
return false;
}
//Point to the Answer portion
unsigned char* reader = &buf[curRecSize];
size_t rLabelSize=0;
if((reader[0]&0xC0) == 0)
{
//Normal label lower 6 bits is the length of the label
rLabelSize=(reader[0]&~(0xC0)*256) + reader[1];
}
else if ((reader[0]&0xC0) == 0xC0)
{
//Compressed label the lower 6 bits and the 8 bits from next octet form a pointer to the compression target.
//Don't need to read it, maybe the same as in Query
rLabelSize=0;
}
else
{
RS_ERR("Answer received with unmanaged label format.");
return false;
}
curRecSize += 2 + rLabelSize;
if (rec_size< static_cast<ssize_t>(curRecSize + sizeof(struct RR_DATA)) )
{
RS_ERR("Request received too small to get Data return.");
return false;
}
//Point to the query portion
struct RR_DATA* rec_data = (struct RR_DATA *)&buf[curRecSize];
if (rec_data->rtype!=qinfo->qtype)
{
RS_ERR("Answer's type received different from query sent.");
return false;
}
if (rec_data->rclass!=qinfo->qclass)
{
RS_ERR("Answer's class received different from query sent.");
return false;
}
curRecSize += sizeof(struct RR_DATA);
if (rec_size< static_cast<ssize_t>(curRecSize + ntohs(rec_data->data_len)) )
{
RS_ERR("Request received too small to get Full Data return.");
return false;
}
//Retrieve Address
if(ntohs(rec_data->data_len)==4)
{
if (isIPV4)
{
in_addr ipv4Add;
ipv4Add.s_addr=*(in_addr_t*)&buf[curRecSize];
#ifdef DEBUG_SPEC_DNS
RS_DBG("Retrieve address: ", rs_inet_ntoa(ipv4Add));
#endif
returned_addr = rs_inet_ntoa(ipv4Add);
return true;
}
}
else if(ntohs(rec_data->data_len)==16)
{
if (!isIPV4)
{
in6_addr ipv6Add;
ipv6Add =*(in6_addr*)&buf[curRecSize];
struct sockaddr_storage ss;
sockaddr_storage_clear(ss);
sockaddr_in6 addr_ipv6;
addr_ipv6.sin6_addr = ipv6Add;
sockaddr_storage_setipv6(ss,&addr_ipv6);
#ifdef DEBUG_SPEC_DNS
RS_DBG("Retrieve address: ", sockaddr_storage_iptostring(ss).c_str());
#endif
returned_addr = sockaddr_storage_iptostring(ss);
return true;
}
}
RS_ERR("Retrieve unmanaged data size=", ntohs(rec_data->data_len));
return false;
}

54
libretroshare/src/util/rserrno.cc
View File

@ -1,54 +0,0 @@
/*******************************************************************************
* libretroshare/src/util: rserrno.cc *
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include <cerrno>
#define RS_INTERNAL_ERRNO_CASE(e) case e: return #e
const char* rsErrnoName(int err)
{
switch (err)
{
RS_INTERNAL_ERRNO_CASE(EINVAL);
RS_INTERNAL_ERRNO_CASE(EBUSY);
RS_INTERNAL_ERRNO_CASE(EAGAIN);
RS_INTERNAL_ERRNO_CASE(EDEADLK);
RS_INTERNAL_ERRNO_CASE(EPERM);
RS_INTERNAL_ERRNO_CASE(EBADF);
RS_INTERNAL_ERRNO_CASE(EFAULT);
RS_INTERNAL_ERRNO_CASE(ENOTSOCK);
RS_INTERNAL_ERRNO_CASE(EISCONN);
RS_INTERNAL_ERRNO_CASE(ECONNREFUSED);
RS_INTERNAL_ERRNO_CASE(ETIMEDOUT);
RS_INTERNAL_ERRNO_CASE(ENETUNREACH);
RS_INTERNAL_ERRNO_CASE(EADDRINUSE);
RS_INTERNAL_ERRNO_CASE(EINPROGRESS);
RS_INTERNAL_ERRNO_CASE(EALREADY);
RS_INTERNAL_ERRNO_CASE(ENOTCONN);
RS_INTERNAL_ERRNO_CASE(EPIPE);
RS_INTERNAL_ERRNO_CASE(ECONNRESET);
RS_INTERNAL_ERRNO_CASE(EHOSTUNREACH);
RS_INTERNAL_ERRNO_CASE(EADDRNOTAVAIL);
}
return "rsErrnoName UNKNOWN ERROR CODE";
}

53
libretroshare/src/util/rsmemory.cc
View File

@ -1,53 +0,0 @@
/*******************************************************************************
* libretroshare/src/util: rsmemory.cc *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012-2012 by Cyril Soler <csoler@users.sourceforge.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "util/rsmemory.h"
void *rs_malloc(size_t size)
{
static const size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024 ; // 1Gb should be enough for everything!
if(size == 0)
{
std::cerr << "(EE) Memory allocation error. A chunk of size 0 was requested. Callstack:" << std::endl;
print_stacktrace() ;
return NULL ;
}
if(size > SAFE_MEMALLOC_THRESHOLD)
{
std::cerr << "(EE) Memory allocation error. A chunk of size larger than " << SAFE_MEMALLOC_THRESHOLD << " was requested. Callstack:" << std::endl;
print_stacktrace() ;
return NULL ;
}
void *mem = malloc(size) ;
if(mem == NULL)
{
std::cerr << "(EE) Memory allocation error for a chunk of " << size << " bytes. Callstack:" << std::endl;
print_stacktrace() ;
return NULL ;
}
return mem ;
}

70
libretroshare/src/util/rsmemory.h
View File

@ -3,8 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2012 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright 2019-2020 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2012 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright (C) 2019-2021 Gioacchino Mazzurco <gio@altermundi.net> *
* Copyright (C) 2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -25,8 +26,10 @@
#include <cstdlib>
#include <iostream>
#include <memory>
#include <system_error>
#include "util/stacktrace.h"
#include "util/rsdebug.h"
/**
* @brief Shorthand macro to declare optional functions output parameters
@ -108,7 +111,66 @@ template<typename T> using rs_view_ptr = T*;
* @see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1408r0.pdf */
template<typename T> using rs_owner_ptr = T*;
void *rs_malloc(size_t size) ;
/// 1Gb should be enough for everything!
static constexpr size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024;
/** Comfortable templated safer malloc, just use it specifing the type of the
* pointer to be returned without need of ugly casting the returned pointer
* `uint8_t* ptr = rs_malloc<uint8_t>(40);`
* @param[in] size number of bytes to allocate
* @param[out] ec optional storage for error details. Value is meaningful only
* whem nullptr is returned.
* @return nullptr on error, pointer to the allocated chuck of memory on success
*/
template<typename T = void> rs_owner_ptr<T> rs_malloc(
size_t size,
rs_view_ptr<std::error_condition> ec = nullptr )
{
if(size == 0)
{
if(!ec)
{
RS_ERR("A chunk of size 0 was requested");
print_stacktrace();
exit(static_cast<int>(std::errc::invalid_argument));
}
*ec = std::errc::invalid_argument;
return nullptr;
}
if(size > SAFE_MEMALLOC_THRESHOLD)
{
if(!ec)
{
RS_ERR( "A chunk of size larger than ", SAFE_MEMALLOC_THRESHOLD,
" was requested" );
exit(static_cast<int>(std::errc::argument_out_of_domain));
}
*ec = std::errc::argument_out_of_domain;
return nullptr;
}
void* mem = malloc(size);
if(!mem)
{
if(!ec)
{
RS_ERR( "Allocation failed for a chunk of ", size,
" bytes with: ", errno);
print_stacktrace();
exit(errno);
}
*ec = rs_errno_to_condition(errno);
return nullptr;
}
return static_cast<rs_owner_ptr<T>>(mem);
}
/** @deprecated use std::unique_ptr instead
// This is a scope guard to release the memory block when going of of the current scope.
@ -128,7 +190,7 @@ void *rs_malloc(size_t size) ;
//
// } // mem gets freed automatically
*/
class RsTemporaryMemory
class RS_DEPRECATED_FOR("std::unique_ptr") RsTemporaryMemory
{
public:
explicit RsTemporaryMemory(size_t s)

14
libretroshare/src/util/rsnet.cc
View File

@ -174,3 +174,17 @@ std::string rs_inet_ntoa(struct in_addr in)
rs_sprintf(str, "%u.%u.%u.%u", (int) bytes[0], (int) bytes[1], (int) bytes[2], (int) bytes[3]);
return str;
}
int rs_setSockTimeout( int sockfd, bool forReceive /*= true*/
, int timeout_Sec /*= 0*/, int timeout_uSec /*= 0*/)
{
#ifdef WINDOWS_SYS
DWORD timeout = timeout_Sec * 1000 + timeout_uSec;
#else
struct timeval timeout;
timeout.tv_sec = timeout_Sec;
timeout.tv_usec = timeout_uSec;
#endif
return setsockopt( sockfd, SOL_SOCKET, forReceive ? SO_RCVTIMEO : SO_SNDTIMEO
, (const char*)&timeout, sizeof timeout);
}

22
libretroshare/src/util/rsnet.h
View File

@ -84,6 +84,18 @@ bool isExternalNet(const struct in_addr *addr);
// uses a re-entrant version of gethostbyname
bool rsGetHostByName(const std::string& hostname, in_addr& returned_addr) ;
/**
* @brief Get hostName address using specific DNS server.
* Using it allow to direct ask our Address to IP, so no need to have a DNS (IPv4 or IPv6).
* If we ask to a IPv6 DNS Server, it respond for our IPv6 address.
* @param servername: Address or name of DNS Server.
* @param hostname: HosteName to get IP ("myip.opendns.com" to get own).
* @param returned_addr: returned IP of hostname.
* @param timeout_s: Timeout in sec to wait server response.
* @return True in success.
*/
bool rsGetHostByNameSpecDNS(const std::string& servername, const std::string& hostname, std::string& returned_addr, int timeout_s = -1);
std::ostream& operator<<(std::ostream& o, const sockaddr_in&);
std::ostream& operator<<(std::ostream& o, const sockaddr_storage&);
@ -159,4 +171,14 @@ bool sockaddr_storage_inet_ntop(const sockaddr_storage &addr, std::string &dst);
int rs_setsockopt( int sockfd, int level, int optname,
const uint8_t *optval, uint32_t optlen );
/**
* @brief Set socket Timeout.
* @param sockfd: The socket to manage.
* @param forReceive: True for Receive, False for Send.
* @param timeout_Sec: Timeout second part.
* @param timeout_uSec: Timeout micro second part.
* @return 0 on success, -1 for errors.
*/
int rs_setSockTimeout( int sockfd, bool forReceive = true, int timeout_Sec = 0, int timeout_uSec = 0);
#endif /* RS_UNIVERSAL_NETWORK_HEADER */

9
libretroshare/src/util/rsnet_ss.cc
View File

@ -233,8 +233,7 @@ bool sockaddr_storage_setport(struct sockaddr_storage &addr, uint16_t port)
bool sockaddr_storage_setipv4(struct sockaddr_storage &addr, const sockaddr_in *addr_ipv4)
{
#ifdef SS_DEBUG
std::cerr << "sockaddr_storage_setipv4()";
std::cerr << std::endl;
RS_ERR();
#endif
sockaddr_storage_clear(addr);
@ -249,7 +248,9 @@ bool sockaddr_storage_setipv4(struct sockaddr_storage &addr, const sockaddr_in *
bool sockaddr_storage_setipv6(struct sockaddr_storage &addr, const sockaddr_in6 *addr_ipv6)
{
std::cerr << "sockaddr_storage_setipv6()" << std::endl;
#ifdef SS_DEBUG
RS_ERR();
#endif
sockaddr_storage_clear(addr);
struct sockaddr_in6 *ipv6_ptr = to_ipv6_ptr(addr);
@ -262,6 +263,7 @@ bool sockaddr_storage_setipv6(struct sockaddr_storage &addr, const sockaddr_in6
}
#ifdef WINDOWS_SYS
#ifndef InetPtonA
int inet_pton(int af, const char *src, void *dst)
{
sockaddr_storage ss;
@ -288,6 +290,7 @@ int inet_pton(int af, const char *src, void *dst)
return 0;
}
#endif
#endif
bool sockaddr_storage_inet_pton( sockaddr_storage &addr,
const std::string& ipStr )

46
libretroshare/src/util/rsthreads.cc
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2004-2007 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2016-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2016-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -25,7 +25,6 @@
#include "rsthreads.h"
#include "util/rsdebug.h"
#include "util/rserrno.h"
#include <chrono>
#include <ctime>
@ -92,11 +91,12 @@ void RsThread::resetTid()
#ifdef WINDOWS_SYS
memset (&mTid, 0, sizeof(mTid));
#else
mTid = 0;
mTid = pthread_t(); //Thread identifiers should be considered opaque and can be null.
#endif
}
RsThread::RsThread() : mHasStopped(true), mShouldStop(false), mLastTid()
RsThread::RsThread() : mInitMtx("RsThread"), mHasStopped(true),
mShouldStop(false), mLastTid()
#ifdef RS_THREAD_FORCE_STOP
, mStopTimeout(0)
#endif
@ -115,19 +115,24 @@ void RsThread::askForStop()
void RsThread::wrapRun()
{
{RS_STACK_MUTEX(mInitMtx);} // Waiting Init done.
run();
resetTid();
mHasStopped = true;
}
void RsThread::fullstop()
{
askForStop();
waitWhileStopping();
}
void RsThread::waitWhileStopping()
{
#ifdef RS_THREAD_FORCE_STOP
const rstime_t stopRequTS = time(nullptr);
#endif
askForStop();
const pthread_t callerTid = pthread_self();
if(pthread_equal(mTid, callerTid))
{
@ -169,7 +174,8 @@ void RsThread::fullstop()
{
RsErr() << __PRETTY_FUNCTION__ << " pthread_cancel("
<< std::hex << mTid << std::dec <<") returned "
<< terr << " " << rsErrnoName(terr) << std::endl;
<< terr << " " << rs_errno_to_condition(terr)
<< std::endl;
print_stacktrace();
}
@ -184,23 +190,15 @@ bool RsThread::start(const std::string& threadName)
// Atomically check if the thread was already started and set it as running
if(mHasStopped.exchange(false))
{
RS_STACK_MUTEX(mInitMtx); // Block thread starting to run
mShouldStop = false;
int pError = pthread_create(
&mTid, nullptr, &rsthread_init, static_cast<void*>(this) );
if(pError)
{
RsErr() << __PRETTY_FUNCTION__ << " pthread_create could not create"
<< " new thread: " << threadName << " pError: " << pError
<< std::endl;
mHasStopped = true;
print_stacktrace();
return false;
}
if(!mTid)
{
RsErr() << __PRETTY_FUNCTION__ << " pthread_create could not create"
<< " new thread: " << threadName << " mTid: " << mTid
<< std::endl;
RS_ERR( "pthread_create could not create new thread: ", threadName,
rs_errno_to_condition(pError) );
mHasStopped = true;
print_stacktrace();
return false;
@ -281,7 +279,7 @@ void RsMutex::lock()
if( err != 0)
{
RsErr() << __PRETTY_FUNCTION__ << "pthread_mutex_lock returned: "
<< rsErrnoName(err)
<< rs_errno_to_condition(err)
#ifdef RS_MUTEX_DEBUG
<< " name: " << name()
#endif
@ -325,7 +323,11 @@ RsThread::~RsThread()
<< "likely to crash because of this." << std::endl;
print_stacktrace();
fullstop();
/* Last resort attempt to stop the thread in a less pathological state.
* Don't call fullstop() as it rely on virtual methods that at this
* point are not anymore the one from inerithing classes causing
* compilers to output a warning */
waitWhileStopping();
}
}

13
libretroshare/src/util/rsthreads.h
View File

@ -4,8 +4,8 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2004-2006 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2016-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* Copyright (C) 2016-2021 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2021 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -105,7 +105,7 @@ class RsStackMutex
{
public:
RsStackMutex(RsMutex &mtx) : mMtx(mtx)
explicit RsStackMutex(RsMutex& mtx): mMtx(mtx)
{
mMtx.lock();
#ifdef RS_MUTEX_DEBUG
@ -261,6 +261,13 @@ private:
/** Call @see run() setting the appropriate flags around it*/
void wrapRun();
/** Wait the thread while it is stopping */
void waitWhileStopping();
/** To be sure Init (pthread_setname_np) is done before continue thread.
* Else can finish before and crash. */
RsMutex mInitMtx;
/// True if thread is stopped, false otherwise
std::atomic<bool> mHasStopped;

3
libretroshare/src/util/rstime.h
View File

@ -3,7 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2013-2013 by Cyril Soler <csoler@users.sourceforge.net> *
* Copyright 2013 Cyril Soler <csoler@users.sourceforge.net> *
* Copyright 2018 Gioacchino Mazzurco <gio@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *

3
plugins/Common/retroshare_plugin.pri
View File

@ -103,7 +103,8 @@ win32 {
for(bin, BIN_DIR):LIBS += -L"$$bin"
LIBS += -lpthread
QMAKE_LFLAGS += -Wl,--end-group
# Do not add for MinGW
!isEmpty(QMAKE_SH): QMAKE_LFLAGS += -Wl,--end-group
}
macx {

23
plugins/FeedReader/gui/AddFeedDialog.ui
View File

@ -6,8 +6,8 @@
<rect>
<x>0</x>
<y>0</y>
<width>715</width>
<height>605</height>
<width>1068</width>
<height>880</height>
</rect>
</property>
<property name="windowTitle">
@ -37,14 +37,14 @@
<widget class="HeaderFrame" name="headerFrame"/>
</item>
<item>
<widget class="QFrame" name="frame">
<widget class="QFrame" name="gradFrame">
<property name="frameShape">
<enum>QFrame::NoFrame</enum>
</property>
<property name="frameShadow">
<enum>QFrame::Raised</enum>
</property>
<layout class="QGridLayout" name="gridLayout_7">
<layout class="QGridLayout" name="gradFrame_GL">
<item row="3" column="0">
<widget class="QGroupBox" name="typeGroupBox">
<property name="title">
@ -73,7 +73,7 @@
</widget>
</item>
<item>
<widget class="QComboBox" name="forumComboBox"/>
<widget class="RSComboBox" name="forumComboBox"/>
</item>
</layout>
</item>
@ -306,6 +306,14 @@
</item>
<item row="1" column="0">
<widget class="QLabel" name="nameLabel">
<property name="font">
<font>
<pointsize>11</pointsize>
<weight>75</weight>
<italic>true</italic>
<bold>true</bold>
</font>
</property>
<property name="text">
<string>Name:</string>
</property>
@ -405,6 +413,11 @@
<header>gui/common/HeaderFrame.h</header>
<container>1</container>
</customwidget>
<customwidget>
<class>RSComboBox</class>
<extends>QComboBox</extends>
<header>gui/common/RSComboBox.h</header>
</customwidget>
</customwidgets>
<tabstops>
<tabstop>urlLineEdit</tabstop>

14
plugins/FeedReader/gui/FeedReaderDialog.ui
View File

@ -86,7 +86,14 @@
</widget>
</item>
<item>
<widget class="StyledLabel" name="titleBarLabel">
<widget class="QLabel" name="titleBarLabel">
<property name="font">
<font>
<pointsize>12</pointsize>
<weight>75</weight>
<bold>true</bold>
</font>
</property>
<property name="text">
<string>Feeds</string>
</property>
@ -178,11 +185,6 @@
</layout>
</widget>
<customwidgets>
<customwidget>
<class>StyledLabel</class>
<extends>QLabel</extends>
<header>gui/common/StyledLabel.h</header>
</customwidget>
<customwidget>
<class>RSTreeWidget</class>
<extends>QTreeWidget</extends>

Some files were not shown because too many files have changed in this diff Show More