.gitignore

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: (C) 2004-2022 Retroshare Team <contact@retroshare.cc>
+# SPDX-FileCopyrightText: (C) 2004-2019 Retroshare Team <contact@retroshare.cc>
 # SPDX-License-Identifier: CC0-1.0
 
 *.o
@@ -15,23 +15,3 @@ Thumbs.db
 *.pro.user
 .kdev4
 *.kdev4
-
-.qmake.stash
-
-!supportlibs/libsam3/Makefile
-
-# QtCreator cruft
-*CMakeLists.txt.user*
-
-# Build artifacts
-/jsonapi-generator/src/jsonapi-generator
-/jsonapi-generator/src/jsonapi-generator-doxygen-final.conf
-/jsonapi-generator/src/jsonapi-includes.inl
-/jsonapi-generator/src/jsonapi-wrappers.inl
-/jsonapi-generator/src/xml/
-/retroshare-friendserver/src/retroshare-friendserver
-/retroshare-gui/src/include/
-/retroshare-gui/src/retroshare
-/retroshare-gui/src/temp/
-/retroshare-service/src/retroshare-service
-/*.tar.?z

.gitlab-ci.yml

@@ -1,50 +1,32 @@
-image: docker:latest
+image: docker:stable
 
 services:
-  - docker:dind
-
-stages:
-  - build
-  - test
+    - docker:stable-dind
 
 workflow:
   rules:
     - if: $CI_MERGE_REQUEST_ID
     - if: $CI_COMMIT_BRANCH
 
-variables:
-  UBUNTU_TESTING_IMAGE_TAG: "$CI_REGISTRY_IMAGE/gitlabci_outputs/ubuntu_testing:$CI_COMMIT_SHA"
-        
-build-ubuntu-test-image:
-    stage: build
+build-and-test:
     script:
-        - docker --version
-        - > 
-          docker login "$CI_REGISTRY"
-          --username "$CI_REGISTRY_USER"
-          --password "$CI_REGISTRY_PASSWORD"
+        - >
+          if [ -n "$CI_MERGE_REQUEST_ID" ]; then
+          REPO_ARGS="--build-arg REPO_URL=$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" ;
+          REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ;
+          else
+          REPO_ARGS="--build-arg REPO_URL=$CI_REPOSITORY_URL" ;
+          REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_COMMIT_BRANCH" ;
+          fi ;
+          export REPO_ARGS ;
+          echo REPO_ARGS=$REPO_ARGS ;
         - mkdir Dockercontext
         - >
-          docker build
-          -t $UBUNTU_TESTING_IMAGE_TAG
-          $($CI_PROJECT_DIR/build_scripts/GitlabCI/getRepoArgs.sh)
+          docker build -t retroshare:testing $REPO_ARGS
           --file $CI_PROJECT_DIR/build_scripts/GitlabCI/gitlabCI.Dockerfile
           Dockercontext
-        - docker push $UBUNTU_TESTING_IMAGE_TAG
-        - echo UBUNTU_TESTING_IMAGE_TAG=$UBUNTU_TESTING_IMAGE_TAG
-
-test-ubuntu:
-    stage: test
-    script:
-        - docker --version
-        - > 
-          docker login "$CI_REGISTRY"
-          --username "$CI_REGISTRY_USER"
-          --password "$CI_REGISTRY_PASSWORD"
-        - echo UBUNTU_TESTING_IMAGE_TAG=$UBUNTU_TESTING_IMAGE_TAG
-        - docker pull $UBUNTU_TESTING_IMAGE_TAG
         - >
-          docker run --name retroshare --detach --tty $UBUNTU_TESTING_IMAGE_TAG
+          docker run --name retroshare --detach --tty retroshare:testing
           retroshare-service --jsonApiPort 9092
         - apk add jq
         - >

.gitmodules

@@ -10,32 +10,7 @@
 [submodule "supportlibs/udp-discovery-cpp"]
 	path = supportlibs/udp-discovery-cpp
 	url = https://github.com/truvorskameikin/udp-discovery-cpp.git
-	branch = master
-# develop branch was removed we were using it at commit f3a3103a6c52e5707629e8d0a7e279a7758fe845
+	branch = develop
 [submodule "supportlibs/rapidjson"]
 	path = supportlibs/rapidjson
 	url = https://github.com/Tencent/rapidjson.git
-[submodule "supportlibs/libsam3"]
-	path = supportlibs/libsam3
-	url = https://github.com/i2p/libsam3.git
-[submodule "supportlibs/jni.hpp"]
-	path = supportlibs/jni.hpp
-	url = https://github.com/RetroShare/jni.hpp.git
-[submodule "openpgpsdk"]
-	path = openpgpsdk
-	url = ../OpenPGP-SDK
-	branch = master
-[submodule "libbitdht"]
-	path = libbitdht
-	url = ../BitDHT
-	branch = master
-[submodule "libretroshare"]
-	path = libretroshare
-	url = https://github.com/RetroShare/libretroshare.git
-	branch = master
-[submodule "retroshare-webui"]
-	path = retroshare-webui
-	url = https://github.com/RetroShare/RSNewWebUI.git
-[submodule "supportlibs/librnp"]
-	path = supportlibs/librnp
-	url = https://github.com/rnpgp/rnp.git

.travis.yml

@@ -8,12 +8,12 @@ language: cpp
 
 matrix:
   include:
-#    - os: linux
-#      dist: bionic
-#      sudo: required
-#      compiler: gcc
+    - os: linux
+      dist: bionic
+      sudo: required
+      compiler: gcc
     - os: osx
-      osx_image: xcode10.2
+      osx_image: xcode10.1
       compiler: clang
       sudo: false
 
@@ -68,11 +68,11 @@ before_script:
     fi
 
 script:
-  - if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then travis_wait 30 make -j$(nproc); fi
+  - if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make -j4; fi
 
 after_success:
   - if [ $TRAVIS_OS_NAME == osx ]; then build_scripts/OSX/travis_makeOSXPackage.sh ; fi
-#  - if [ $TRAVIS_OS_NAME == linux ] && [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make -j2; fi
+  - if [ $TRAVIS_OS_NAME == linux ] && [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make -j2; fi
 
 
 # branches:

README.asciidoc

@@ -10,21 +10,10 @@ RetroShare provides file sharing, chat, messages, forums, channels and more.
 .Build Status
 |===============================================================================
 |GNU/Linux (via Gitlab CI) | image:https://gitlab.com/RetroShare/RetroShare/badges/master/pipeline.svg[link="https://gitlab.com/RetroShare/RetroShare/-/commits/master",title="pipeline status"]
+|GNU/Linux, macOS, (via Travis CI) | image:https://travis-ci.org/RetroShare/RetroShare.svg?branch=master[link="https://travis-ci.org/RetroShare/RetroShare"]
 |Windows (via AppVeyor) | image:https://ci.appveyor.com/api/projects/status/github/RetroShare/RetroShare?svg=true[link="https://ci.appveyor.com/project/RetroShare58622/retroshare"]
-|macOS, (via Travis CI) | image:https://app.travis-ci.com/RetroShare/RetroShare.svg?branch=master[link="https://app.travis-ci.com/github/RetroShare/RetroShare"]
-|libretroshare GNU/Linux, Android (via Gitlab CI) | image:https://gitlab.com/RetroShare/libretroshare/badges/master/pipeline.svg[link="https://gitlab.com/RetroShare/libretroshare/-/commits/master",title="pipeline status"]
 |===============================================================================
 
-== Get the source
-
-Clone this repository, enter the directory and then get the submodules with the
-following command
-
-[source,bash]
---------
-git submodule update --init --remote --force libbitdht/ libretroshare/ openpgpsdk/
---------
-
 
 == Compilation
 

RetroShare.pro

@@ -25,6 +25,9 @@ CONFIG += c++14
 
 TEMPLATE = subdirs
 
+SUBDIRS += openpgpsdk
+openpgpsdk.file = openpgpsdk/src/openpgpsdk.pro
+
 rs_jsonapi:isEmpty(JSONAPI_GENERATOR_EXE) {
     SUBDIRS += jsonapi-generator
     jsonapi-generator.file = jsonapi-generator/src/jsonapi-generator.pro
@@ -33,7 +36,7 @@ rs_jsonapi:isEmpty(JSONAPI_GENERATOR_EXE) {
 
 SUBDIRS += libbitdht
 libbitdht.file = libbitdht/src/libbitdht.pro
-libretroshare.depends += libbitdht
+libretroshare.depends += openpgpsdk libbitdht
 
 SUBDIRS += libretroshare
 libretroshare.file = libretroshare/src/libretroshare.pro
@@ -52,12 +55,6 @@ retroshare_service {
     retroshare_service.target = retroshare_service
 }
 
-retroshare_friendserver {
-    SUBDIRS += retroshare_friendserver
-    retroshare_friendserver.file = retroshare-friendserver/src/retroshare-friendserver.pro
-    retroshare_friendserver.depends = libretroshare
-    retroshare_friendserver.target = retroshare_friendserver
-}
 retroshare_plugins {
     SUBDIRS += plugins
     plugins.file = plugins/plugins.pro
@@ -65,13 +62,6 @@ retroshare_plugins {
     plugins.target = plugins
 }
 
-rs_webui {
-    SUBDIRS += retroshare-webui
-    retroshare-webui.file = retroshare-webui/webui.pro
-    retroshare-webui.target = rs_webui
-    retroshare-webui.depends = retroshare_gui
-}
-
 wikipoos {
     SUBDIRS += pegmarkdown
     pegmarkdown.file = supportlibs/pegmarkdown/pegmarkdown.pro

appveyor.yml

@@ -16,7 +16,6 @@
 
 # version format
 version: RetroShare-git-{branch}-{build}
-image: Visual Studio 2019
 
 # you can use {branch} name in version format too
 # version: 1.0.{build}-{branch}
@@ -95,7 +94,7 @@ install:
   - set PATH=C:\msys64\usr\bin;%PATH%
   - set PATH=C:\msys64\mingw32\bin;%PATH%
   - >
-    pacman --noconfirm -Sy
+    pacman --noconfirm -S
     mingw-w64-i686-qt5 mingw-w64-i686-miniupnpc mingw-w64-i686-rapidjson
     mingw-w64-i686-sqlcipher mingw-w64-i686-xapian-core mingw-w64-i686-cmark
   #- set PATH=C:\msys64\mingw32\qt5-static\bin\;%PATH%
@@ -258,10 +257,10 @@ after_build:
 #      artifacts configuration    #
 #---------------------------------#
 
-artifacts:
-  - path: '**\*.exe'
-  - path: filelist.txt
-  - path: $(RS_DEPLOY)
+#artifacts:
+#  - path: '**\*.exe'
+#  - path: filelist.txt
+# - path: $(RS_DEPLOY)
 
 #
 #  # pushing a single file

build_scripts/Android/README.asciidoc

@@ -0,0 +1,399 @@
+= RetroShare development on Android
+
+// SPDX-FileCopyrightText: RetroShare Team <contact@retroshare.cc>
+// SPDX-License-Identifier: CC-BY-SA-4.0
+
+
+Compiling an application for Android is not as easy as one would imagine,
+expecially one like RetroShare that has a big codebase and is not well
+documented. This document is aimed to empower the reader so she can hopefully
+succed or at least have a significant help in compiling her own RetroShare APK
+installable on Android.
+
+
+== Preparing The Environement
+
+First of all setup your Qt for Android development environement following the
+guide on the link:http://doc.qt.io/qt-5/androidgs.html[Qt for android web site].
+At this point you should have Android SDK, Android NDK, and Qt for Android
+working fine, and you should be capable of executing on an Android emulator or
+on your Android phone Qt for Android examples.
+
+But RetroShare is not as simple to compile as those examples. The good news is
+that Android NDK ships all the necessary to build a custom toolchain that is
+suitable to build RetroShare.
+In order to build the toolchain with needed libraries RetroShare provides the
++android-prepare-toolchain.sh+ script; before you execute it you should define
+some variables the script cannot determine in an easy and reliable manner by
+itself in your terminal.
+
+[source,bash]
+-------------------------------------------------------------------------------
+## The path where Android NDK is installed in your system
+export ANDROID_NDK_PATH="/opt/android-ndk/"
+
+## The path where your fresh compiled toolchain will be installed, make sure
+## the parent exists
+export NATIVE_LIBS_TOOLCHAIN_PATH="${HOME}/Builds/android-toolchains/retroshare-android/"
+
+## The CPU architecture of the Android device you want to target
+export ANDROID_NDK_ARCH="arm"
+
+## The Android API level the Android device you want to target
+export ANDROID_PLATFORM_VER="16"
+
+## The number of core that yout host CPU have (just to speed up compilation) set
+## it to 1 if unsure
+export HOST_NUM_CPU=1
+
+./android-prepare-toolchain.sh
+-------------------------------------------------------------------------------
+
+
+== Preparing Qt Creator
+
+Now that your environement is set up you should configure Qt Creator for Android
+following the
+link:http://doc.qt.io/qtcreator/creator-developing-android.html[official guide].
+At the end of this step your Qt Creator should recognize the Android compiler
+and the Qt for Android kit.
+
+Your Kit is now ready to use. Now you can open RetroShare as a Qt Creator
+project and in the Projects left menu add the newly created kit if not already
+present, so you can select it on the build type selection button down on the
+left.
+
+Now you need to set properly a few options like `JSONAPI_GENERATOR_EXE` and
+disable some of RetroShare modules like `retroshare-gui` that are not available
+on Android so you will have to go to
+
+_Qt Creator left pane -> Projects -> Build and Run -> Android SOMESTUFF kit ->
+Build Steps -> qmake -> Additional arguments_  +
+
+and add the following configurations (change `Your_Path` according to your
+deployment)
+
+[source,makefile]
+-------------------------------------------------------------------------------
+CONFIG+=retroshare_service CONFIG+=rs_jsonapi CONFIG+=rs_deep_search
+RS_UPNP_LIB=miniupnpc
+JSONAPI_GENERATOR_EXE=Your_Path/jsonapi-generator/src/jsonapi-generator
+NATIVE_LIBS_TOOLCHAIN_PATH=Your_Path/retroshare-android-16-arm/
+CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password
+CONFIG+=no_rs_service_terminal_login
+-------------------------------------------------------------------------------
+
+TIP: Some versions of QtCreator try to find the Android SDK in
+`/opt/android/sdk`. A workaround to this is to make a symbolic link there
+pointing to your SDK installation path, like
++mkdir -p /opt/android/sdk && ln -s /home/user/android-sdk-linux
+/opt/android/sdk+
+
+
+== Quircks
+
+=== Protected Apps
+
+On some Android devices like +Huawei ALE-L21+ background applications are
+killed when screen is turned off unless they are in the _protected app_ list.
+At moment seems apps developers don't have a way to have the application
+_protected_ by default, unless the phone vendor decide the app is _popular_ so
+the user have to enable _protection_ for RetroShare manually on those mobile
+phones. +
+
+{empty} +
+To enable enable _protection_: +Android menu -> Settings -> Privacy & security 
+-> Protected apps -> RetroShare+ +
+{empty} +
+
+Other devices may offer similar _features_ please report them.
+
+
+=== APK signature mismatch
+
+If you try to install a RetroShare APK that comes from a different source
+(eg: if you try to upgrade from F-Droid when you originally installed an APK
+build by yourself) Android will prevent that from happening. In that case the
+only solution is to uninstall the app and then install the new APK but if you do
+it also the application data and your precious cryptographic keys, friend list
+etc. will be lost forever.
+To avoid that you can attempt to manually backup and then restore from the
+command-line (`adb backup` seems not working either) to change the app source
+without erasing the appliation data.
+
+CAUTION: Following steps require root access on your Android device
+
+.Backup RetroShare Android application data
+[source,bash]
+--------------------------------------------------------------------------------
+export ORIG_DIR="/data/data/org.retroshare.android.qml_app"
+export BACKUP_DIR="org.retroshare.android.qml_app.backup"
+
+adb root
+
+adb shell am force-stop org.retroshare.android.qml_app
+sleep 1s
+
+mkdir ${BACKUP_DIR}
+
+# Avoid adb pull failing
+adb shell rm ${ORIG_DIR}/files/.retroshare/libresapi.sock
+adb pull ${ORIG_DIR}/files/ ${BACKUP_DIR}/files/
+--------------------------------------------------------------------------------
+
+After this you should be able to uninstall the old APK with your preferred
+method, as example from the command-line.
+
+.Uninstall RetroShare Android from the command-line
+[source,bash]
+--------------------------------------------------------------------------------
+adb uninstall org.retroshare.android.qml_app
+--------------------------------------------------------------------------------
+
+Now you can install a different signature APK and then restore the application
+data with the following commands.
+
+[source,bash]
+--------------------------------------------------------------------------------
+export ORIG_DIR="/data/data/org.retroshare.android.qml_app"
+export BACKUP_DIR="org.retroshare.android.qml_app.backup"
+
+adb root
+
+## Launch the app to make sure the parent directory exists and has proper owner
+adb shell monkey -p org.retroshare.android.qml_app -c android.intent.category.LAUNCHER 1
+sleep 1s
+
+adb shell am force-stop org.retroshare.android.qml_app
+sleep 1s
+
+
+APP_OWNER="$(adb shell busybox ls -lnd ${ORIG_DIR} | awk '{print $3":"$4}')"
+adb shell rm -rf ${ORIG_DIR}/files
+adb push ${BACKUP_DIR}/files/ ${ORIG_DIR}/files/
+adb shell busybox chown -R ${APP_OWNER} ${ORIG_DIR}/files/
+--------------------------------------------------------------------------------
+
+Opening RetroShare android app now should show your old profile.
+
+
+== Debugging with GDB
+
+If building RetroShare Android package seems tricky, setting up a functional
+debugging environement for it feels like black magic. This section is meant to
+help you doing it with less headache and hopefully in a reproducible way.
+
+Unfortunately at the time of the last update to this guide, Qt build system
+strips debugging symbols from the package and from the buildroot also if you
+compile with debugging enabled. Fiddling with `qmake` configurations and
+variables like `CONFIG+=debug`, `CONFIG+=force_debug_info` or `QMAKE_STRIP`
+either as commandline arguments or inside retroshare `.pro` and `.pri` files is
+uneffective. IMHO Qt should handle this on itself so it is probably worth
+reporting a bug to them. So to workaround this problem you need to fiddle a bit
+with the Android NDK. In my case I always keep +Debug+ or +Release+ suffix in
+my build directory name depending on what kind of build it is, so I use this
+information and modify `llvm-strip` in a way that it will strip only if stripped
+file path doesn't contain +Debug+.
+
+.Modify llvm-strip inside Android NDK
+--------------------------------------------------------------------------------
+## Set ANDROID_NDK_PATH to your Android NDK installation path
+export ANDROID_NDK_PATH="/opt/android-ndk/"
+
+## Define a convenience variable with llvm-strip path
+export ANDROID_NDK_LLVM_STRIP="${ANDROID_NDK_PATH}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip"
+
+## If not existing yer create a backup of the original llvm-strip
+[ -f "${ANDROID_NDK_LLVM_STRIP}.back" ] ||
+	cp "${ANDROID_NDK_LLVM_STRIP}" "${ANDROID_NDK_LLVM_STRIP}.back"
+
+## Create a new llvm-strip that does nothing if the argument path contains Debug
+cat > "${ANDROID_NDK_LLVM_STRIP}" << LLVMSTRIPTRICK
+#!/bin/bash
+
+echo "\${2}" | grep -q Debug ||
+	"${ANDROID_NDK_LLVM_STRIP}.back" --strip-all "\${2}"
+
+LLVMSTRIPTRICK
+
+## Eventually you can revert back simply by running
+# `mv "${ANDROID_NDK_LLVM_STRIP}.back" "${ANDROID_NDK_LLVM_STRIP}"`
+--------------------------------------------------------------------------------
+
+
+To attach to the `retroshare-service` running on Android you need also to pull a
+debugging sysroot out of your device first, RetroShare sources provides an
+helper script to do that.
+
+.Prepare debugging sysroot with helper script
+[source,bash]
+--------------------------------------------------------------------------------
+## Set RetroShare source path
+export RS_SOURCE_DIR="${HOME}/Development/rs-develop"
+
+## Optionally set your device ID first available will be used, you can run
+## `adb devices` to list available devices.
+#export ANDROID_SERIAL="YT013PSPGK"
+
+## Optionally set a path where to save the debugging sysroot otherwise default
+## is used.
+#export DEBUG_SYSROOT="${HOME}/Builds/debug_sysroot/${ANDROID_SERIAL}/"
+
+## Run the helper script
+${RS_SOURCE_DIR}/build_scripts/Android/pull_sysroot.sh
+--------------------------------------------------------------------------------
+
+.Prepare Android NDK GDB configurations
+[source,bash]
+--------------------------------------------------------------------------------
+## Optionally set Qt version variable consistently with your installation
+export QT_VERSION="5.12.7"
+
+## Optionally set Qt architecture variable consistently with Android device
+export QT_ARCH="arm64_v8a"
+
+## Optionally set Qt path variable consistently with your installation
+export QT_DIR="/opt/Qt-${QT_VERSION}/${QT_VERSION}/"
+
+## Optionally set RetroShare buildroot path
+export RS_BUILD_DIR="${HOME}/Builds/RetroShare-Android_for_${QT_ARCH}_Clang_Qt_${QT_VERSION//./_}_android_${QT_ARCH}-Debug/"
+
+## Optionally set gdb config file path
+export GDB_CONFIGS_FILE="${HOME}/Builds/gdb_configs_${QT_ARCH}"
+
+## Generate Android NDK GDB configuration
+${RS_SOURCE_DIR}/build_scripts/Android/generate_gdb_init_commands.sh
+--------------------------------------------------------------------------------
+
+
+You will need to run the following steps everytime you want to debug
+`retroshare-service` on Android.
+
+Make sure `retroshare-service` is running on your connected Android device.
+
+.Run GDB server on your Android device from your host console
+[source,bash]
+--------------------------------------------------------------------------------
+${RS_SOURCE_DIR}/build_scripts/Android/start_gdbserver.sh
+--------------------------------------------------------------------------------
+
+
+.Run Android NDK GDB on your workstation and attach
+[source,bash]
+--------------------------------------------------------------------------------
+## Start NDK gdb
+${ANDROID_NDK_PATH}/prebuilt/linux-x86_64/bin/gdb
+
+## Instruct GDB how and where to find debugging symbols
+(gdb) source $GDB_CONFIGS_FILE
+
+## Connect to the gdbserver running on the phone
+(gdb) target remote 127.0.01:5039
+
+## Have fun debugging
+(gdb)
+--------------------------------------------------------------------------------
+
+
+== Debugging with Qt Creator
+
+WARNING: As of the last update to this guide, debugging retroshare-service
+running on Android via Qt creator doesn't wrok even with all the trickery
+explained in this section, you better learn how to debug with GDB reading
+carefully previous section.
+
+Qt Creator actually support debugging only for the foreground activity, so to
+debug what's happening in the core extra trickery is needed.
+
+- Run the App in Debug mode from QtCreator "Start Debugging" button
+- Enable QtCreator debugger console view Menu->Window->Debugger Log
+- Run +show solib-search-path+ in the QtCreator GDB console
+- Take note of the output you get in the right pane of the console
+- Thanks https://stackoverflow.com/a/31164313 for the idea
+
+TIP: QtCreator GDB console seems a bit buggy and easly trigger timeout
+message when a command is run, in that case attempt to run the command while the
+debugging is paused or at breakpoint, or with other similar tricks.
+
+CAUTION: Following steps require root access on your Android device
+
+Now on your phone yuo need to authorize root access to applications, then once
+you plug your sacrifical Android phone run this commands
+
+.Run gdbserver as root on Android phone
+[source,bash]
+--------------------------------------------------------------------------------
+## Open a shell from your workstation on the connected Android phone
+adb shell
+
+## take the note of the phone IP address
+ip address show
+
+## Gain root permissions on the shell
+su
+
+## Attach with gdbserver and listen on one TCP port
+gdbserver :4567 --attach $(pgrep org.retroshare.android.qml_app:rs)
+--------------------------------------------------------------------------------
+
+
+.Prepare and run Android NDK GDB on your workstation
+[source,bash]
+--------------------------------------------------------------------------------
+## Setup some convenience variables
+NDK_GDB="${ANDROID_NDK_PATH}/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/bin/arm-linux-androideabi-gdb"
+RS_BUILD_PATH="${HOME}/Builds/RetroShare-Android_for_armeabi_v7a_GCC_4_9_Qt_5_9_2_android_armv7-Debug/"
+
+## Start Android NDK GDB of your phone architecture passing the executable
+$NDK_GDB $RS_BUILD_PATH/retroshare-android-service/src/libretroshare-android-service.so
+
+## Instruct GDB how and where to find debugging symbols
+(gdb) set auto-solib-add on
+(gdb) set solib-search-path THE:BIG:LIST:OF:DIRECTORIES:YOU:TAKE:NOTE:BEFORE
+
+## Connect to the gdbserver running on the phone
+(gdb) target remote $PHONE_IP:4567
+
+## Have fun debugging
+(gdb)
+--------------------------------------------------------------------------------
+
+TIP: Some time WiFi power saving on Android mess with the GDB connection,
+to prevent that from appening open another +adb shell+ and live +ping+ toward
+your work-station running
+
+== Embedding into other Android packages
+
+As showed by https://elrepo.io/[elRepo.io] developers it is possible and quite
+easy to embed `retroshare-service` into other Android packages see description
+
+https://gitlab.com/elRepo.io/elRepo.io-android/-/blob/master/README.adoc
+
+And implementation details
+
+https://gitlab.com/elRepo.io/elRepo.io-android/-/blob/master/android/app/build.gradle
+
+
+== Furter Readings
+
+- link:http://doc.qt.io/qt-5/android-support.html[]
+- link:https://developer.android.com/ndk/guides/libs.html[]
+- link:retroshare://forum?name=Compiling%20nogui%20for%20android&id=8fd22bd8f99754461e7ba1ca8a727995&msgid=4e0f92330600bba9cf978f384f4b7b2f2ca64eff[]
+- link:retroshare://file?name=Android%20Native%20Development%20Kit%20Cookbook.pdf&size=29214468&hash=0123361c1b14366ce36118e82b90faf7c7b1b136[]
+- link:https://groups.google.com/forum/#!topic/android-developers/srATPaL0aRU[]
+- link:https://stackoverflow.com/questions/31638986/protected-apps-setting-on-huawei-phones-and-how-to-handle-it[]
+- link:https://tthtlc.wordpress.com/2012/09/19/how-to-do-remote-debugging-via-gdbserver-running-inside-the-android-phone/[]
+- link:https://source.android.com/devices/tech/debug/[]
+- link:https://source.android.com/devices/tech/debug/gdb[]
+- link:https://fw4spl-org.github.io/fw4spl-blog/2015/07/27/Native-debugging-on-Android-with-QtCreator.html[]
+- link:https://fragglet.livejournal.com/19646.html[]
+- link:https://github.com/android-ndk/ndk/issues/773[How to build without using standalone toolchain?]
+
+== License
+
+Copyright (C) 2016-2020  Gioacchino Mazzurco <gio@eigenlab.org> +
+Copyright (C) 2020       Asociación Civil Altermundi <info@altermundi.net> +
+
+This work is licensed under a Creative Commons Attribution-Share Alike 4.0 International License.
+
+image::https://i.creativecommons.org/l/by-sa/4.0/88x31.png[Creative Commons License, link=http://creativecommons.org/licenses/by-sa/4.0/]

build_scripts/Android/generate_gdb_init_commands.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+# Script to prepare Android NDK GDB configurations to debug retroshare-service
+#
+# Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>
+# Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation, version 3.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>
+
+
+## Define default value for variable, take two arguments, $1 variable name,
+## $2 default variable value, if the variable is not already define define it
+## with default value.
+function define_default_value()
+{
+	VAR_NAME="${1}"
+	DEFAULT_VALUE="${2}"
+
+	[ -z "${!VAR_NAME}" ] && export ${VAR_NAME}="${DEFAULT_VALUE}"
+}
+
+define_default_value QT_VERSION "5.12.4"
+define_default_value QT_ARCH "arm64_v8a"
+define_default_value QT_DIR "/opt/Qt-${QT_VERSION}/${QT_VERSION}/"
+define_default_value ANDROID_SERIAL "$(adb devices | head -n 2 | tail -n 1 | awk '{print $1}')"
+define_default_value RS_BUILD_DIR "${HOME}/Builds/RetroShare-Android_for_${QT_ARCH}_Clang_Qt_${QT_VERSION//./_}_android_${QT_ARCH}-Debug/"
+define_default_value RS_SOURCE_DIR "${HOME}/Development/rs-develop/"
+define_default_value DEBUG_SYSROOT "${HOME}/Builds/debug_sysroot/${ANDROID_SERIAL}/"
+define_default_value GDB_CONFIGS_FILE "${HOME}/Builds/gdb_configs_${QT_ARCH}"
+
+scanDir()
+{
+	find "$1" -type d -not -path '*/\.git/*' | tr '\n' ':' >> $GDB_CONFIGS_FILE
+}
+
+putSeparator()
+{
+	echo >> $GDB_CONFIGS_FILE
+	echo >> $GDB_CONFIGS_FILE
+}
+
+echo "set sysroot ${DEBUG_SYSROOT}" > $GDB_CONFIGS_FILE
+putSeparator
+
+echo "set auto-solib-add on" >> $GDB_CONFIGS_FILE
+echo -n "set solib-search-path " >> $GDB_CONFIGS_FILE
+scanDir "${RS_BUILD_DIR}"
+scanDir "${DEBUG_SYSROOT}"
+scanDir "${QT_DIR}/android_${QT_ARCH}/lib/"
+scanDir "${QT_DIR}/android_${QT_ARCH}/plugins/"
+scanDir "${QT_DIR}/android_${QT_ARCH}/qml/"
+putSeparator
+
+echo -n "directory " >> $GDB_CONFIGS_FILE
+scanDir ${RS_SOURCE_DIR}/jsonapi-generator/src
+scanDir ${RS_SOURCE_DIR}/libbitdht/src
+scanDir ${RS_SOURCE_DIR}/openpgpsdk/src
+scanDir ${RS_SOURCE_DIR}/libretroshare/src
+scanDir ${RS_SOURCE_DIR}/retroshare-service/src
+scanDir ${RS_SOURCE_DIR}/supportlibs/rapidjson/include/
+scanDir ${RS_SOURCE_DIR}/supportlibs/restbed/source/
+scanDir ${RS_SOURCE_DIR}/supportlibs/udp-discovery-cpp/
+scanDir ${RS_SOURCE_DIR}/supportlibs/restbed/dependency/asio/asio/include/
+scanDir ${RS_SOURCE_DIR}/supportlibs/restbed/dependency/catch/include/
+putSeparator
+
+## see https://stackoverflow.com/questions/28972367/gdb-backtrace-without-stopping
+echo "catch signal SIGSEGV" >> $GDB_CONFIGS_FILE
+echo "commands
+   bt
+   continue
+   end" >> $GDB_CONFIGS_FILE
+putSeparator
+
+echo GDB_CONFIGS_FILE=$GDB_CONFIGS_FILE

build_scripts/Android/prepare-toolchain-clang.sh

@@ -0,0 +1,549 @@
+#!/bin/bash
+
+# Script to prepare RetroShare Android package building toolchain
+#
+# Copyright (C) 2016-2020  Gioacchino Mazzurco <gio@eigenlab.org>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation, version 3.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>
+#
+# SPDX-FileCopyrightText: Retroshare Team <contact@retroshare.cc>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+
+## Define default value for variable, take two arguments, $1 variable name,
+## $2 default variable value, if the variable is not already define define it
+## with default value.
+function define_default_value()
+{
+	VAR_NAME="${1}"
+	DEFAULT_VALUE="${2}"
+
+	[ -z "${!VAR_NAME}" ] && export ${VAR_NAME}="${DEFAULT_VALUE}"
+}
+
+## You are supposed to provide the following variables according to your system setup
+define_default_value ANDROID_NDK_PATH "/opt/android-ndk/"
+define_default_value ANDROID_NDK_ARCH "arm"
+define_default_value ANDROID_PLATFORM_VER "16"
+define_default_value NATIVE_LIBS_TOOLCHAIN_PATH "${HOME}/Builds/android-toolchains/retroshare-android-${ANDROID_PLATFORM_VER}-${ANDROID_NDK_ARCH}/"
+define_default_value HOST_NUM_CPU $(nproc)
+
+define_default_value BZIP2_SOURCE_VERSION "1.0.6"
+define_default_value BZIP2_SOURCE_SHA256 a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
+
+define_default_value OPENSSL_SOURCE_VERSION "1.1.1c"
+define_default_value OPENSSL_SOURCE_SHA256 f6fb3079ad15076154eda9413fed42877d668e7069d9b87396d0804fdb3f4c90
+
+define_default_value SQLITE_SOURCE_YEAR "2018"
+define_default_value SQLITE_SOURCE_VERSION "3250200"
+define_default_value SQLITE_SOURCE_SHA256 da9a1484423d524d3ac793af518cdf870c8255d209e369bd6a193e9f9d0e3181
+
+define_default_value SQLCIPHER_SOURCE_VERSION "4.2.0"
+define_default_value SQLCIPHER_SOURCE_SHA256 105c1b813f848da038c03647a8bfc9d42fb46865e6aaf4edfd46ff3b18cdccfc
+
+define_default_value LIBUPNP_SOURCE_VERSION "1.8.4"
+define_default_value LIBUPNP_SOURCE_SHA256 976c3e4555604cdd8391ed2f359c08c9dead3b6bf131c24ce78e64d6669af2ed
+
+define_default_value INSTALL_QT_ANDROID "false"
+define_default_value QT_VERSION "5.12.0"
+define_default_value QT_ANDROID_INSTALLER_SHA256 a214084e2295c9a9f8727e8a0131c37255bf724bfc69e80f7012ba3abeb1f763
+
+define_default_value RESTBED_SOURCE_VERSION f74f9329dac82e662c1d570b7cd72c192b729eb4
+
+define_default_value UDP_DISCOVERY_CPP_SOURCE "https://github.com/truvorskameikin/udp-discovery-cpp.git"
+define_default_value UDP_DISCOVERY_CPP_VERSION "develop"
+
+define_default_value XAPIAN_SOURCE_VERSION "1.4.7"
+define_default_value XAPIAN_SOURCE_SHA256 13f08a0b649c7afa804fa0e85678d693fd6069dd394c9b9e7d41973d74a3b5d3
+
+define_default_value RAPIDJSON_SOURCE_VERSION "1.1.0"
+define_default_value RAPIDJSON_SOURCE_SHA256 bf7ced29704a1e696fbccf2a2b4ea068e7774fa37f6d7dd4039d0787f8bed98e
+
+define_default_value MINIUPNPC_SOURCE_VERSION "2.1.20190625"
+define_default_value MINIUPNPC_SOURCE_SHA256 8723f5d7fd7970de23635547700878cd29a5c2bb708b5e5475b2d1d2510317fb
+
+
+## $1 filename, $2 sha256 hash
+function check_sha256()
+{
+	echo ${2} "${1}" | sha256sum -c &> /dev/null
+}
+
+## $1 filename, $2 sha256 hash, $3 url
+function verified_download()
+{
+	FILENAME="$1"
+	SHA256="$2"
+	URL="$3"
+
+	check_sha256 "${FILENAME}" "${SHA256}" ||
+	{
+		rm -rf "${FILENAME}"
+
+		wget -O "${FILENAME}" "$URL" ||
+		{
+			echo "Failed downloading ${FILENAME} from $URL"
+			exit 1
+		}
+
+		check_sha256 "${FILENAME}" "${SHA256}" ||
+		{
+			echo "SHA256 mismatch for ${FILENAME} from ${URL} expected sha256 ${SHA256} got $(sha256sum ${FILENAME} | awk '{print $1}')"
+			exit 1
+		}
+	}
+}
+
+cArch=""
+eABI=""
+
+case "${ANDROID_NDK_ARCH}" in
+"arm")
+	cArch="${ANDROID_NDK_ARCH}"
+	eABI="eabi"
+	;;
+"arm64")
+	cArch="aarch64"
+	eABI=""
+	;;
+"x86")
+	cArch="i686"
+	eABI=""
+	;;
+esac
+
+export SYSROOT="${NATIVE_LIBS_TOOLCHAIN_PATH}/sysroot/"
+export PREFIX="${SYSROOT}/usr/"
+export CC="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang"
+export CXX="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang++"
+export AR="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ar"
+export RANLIB="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ranlib"
+
+
+## More information available at https://android.googlesource.com/platform/ndk/+/ics-mr0/docs/STANDALONE-TOOLCHAIN.html
+build_toolchain()
+{
+	echo "build_toolchain()
+################################################################################
+################################################################################
+################################################################################
+"
+
+	rm -rf ${NATIVE_LIBS_TOOLCHAIN_PATH}
+	${ANDROID_NDK_PATH}/build/tools/make_standalone_toolchain.py --verbose \
+		--arch ${ANDROID_NDK_ARCH} --install-dir ${NATIVE_LIBS_TOOLCHAIN_PATH} \
+		--api ${ANDROID_PLATFORM_VER}
+	find "${PREFIX}/include/" -not -type d > "${NATIVE_LIBS_TOOLCHAIN_PATH}/deletefiles"
+}
+
+## This avoid <cmath> include errors due to -isystem and -I ordering issue
+delete_copied_includes()
+{
+	echo "delete_copied_includes()
+################################################################################
+################################################################################
+################################################################################
+"
+	cat "${NATIVE_LIBS_TOOLCHAIN_PATH}/deletefiles" | while read delFile ; do
+		rm "$delFile"
+	done
+}
+
+## More information available at https://gitlab.com/relan/provisioners/merge_requests/1 and http://stackoverflow.com/a/34032216
+install_qt_android()
+{
+	echo "install_qt_android()
+################################################################################
+################################################################################
+################################################################################
+"
+
+	QT_VERSION_CODE=$(echo $QT_VERSION | tr -d .)
+	QT_INSTALL_PATH=${NATIVE_LIBS_TOOLCHAIN_PATH}/Qt
+	QT_INSTALLER="qt-unified-linux-x64-3.0.2-online.run"
+
+	verified_download $QT_INSTALLER $QT_ANDROID_INSTALLER_SHA256 \
+		http://master.qt.io/archive/online_installers/3.0/${QT_INSTALLER}
+
+	chmod a+x ${QT_INSTALLER}
+
+	QT_INSTALLER_SCRIPT="qt_installer_script.js"
+	cat << EOF > "${QT_INSTALLER_SCRIPT}"
+function Controller() {
+    installer.autoRejectMessageBoxes();
+    installer.installationFinished.connect(function() {
+        gui.clickButton(buttons.NextButton);
+    });
+
+    var welcomePage = gui.pageWidgetByObjectName("WelcomePage");
+    welcomePage.completeChanged.connect(function() {
+        if (gui.currentPageWidget().objectName == welcomePage.objectName)
+            gui.clickButton(buttons.NextButton);
+    });
+}
+
+Controller.prototype.WelcomePageCallback = function() {
+    gui.clickButton(buttons.NextButton);
+}
+
+Controller.prototype.CredentialsPageCallback = function() {
+    gui.clickButton(buttons.NextButton);
+}
+
+Controller.prototype.IntroductionPageCallback = function() {
+    gui.clickButton(buttons.NextButton);
+}
+
+Controller.prototype.TargetDirectoryPageCallback = function() {
+    gui.currentPageWidget().TargetDirectoryLineEdit.setText("$QT_INSTALL_PATH");
+    gui.clickButton(buttons.NextButton);
+}
+
+Controller.prototype.ComponentSelectionPageCallback = function() {
+    var widget = gui.currentPageWidget();
+
+    // You can get these component names by running the installer with the
+    // --verbose flag. It will then print out a resource tree.
+
+    widget.deselectComponent("qt.tools.qtcreator");
+    widget.deselectComponent("qt.tools.doc");
+    widget.deselectComponent("qt.tools.examples");
+
+    widget.selectComponent("qt.$QT_VERSION_CODE.android_armv7");
+
+    gui.clickButton(buttons.NextButton);
+}
+
+Controller.prototype.LicenseAgreementPageCallback = function() {
+    gui.currentPageWidget().AcceptLicenseRadioButton.setChecked(true);
+    gui.clickButton(buttons.NextButton);
+}
+
+Controller.prototype.StartMenuDirectoryPageCallback = function() {
+    gui.clickButton(buttons.NextButton);
+}
+
+Controller.prototype.ReadyForInstallationPageCallback = function() {
+    gui.clickButton(buttons.NextButton);
+}
+
+Controller.prototype.FinishedPageCallback = function() {
+    var checkBoxForm = gui.currentPageWidget().LaunchQtCreatorCheckBoxForm;
+    if (checkBoxForm && checkBoxForm.launchQtCreatorCheckBox)
+        checkBoxForm.launchQtCreatorCheckBox.checked = false;
+    gui.clickButton(buttons.FinishButton);
+}
+EOF
+
+QT_QPA_PLATFORM=minimal ./${QT_INSTALLER} --script ${QT_INSTALLER_SCRIPT}
+}
+
+## More information available at retroshare://file?name=Android%20Native%20Development%20Kit%20Cookbook.pdf&size=29214468&hash=0123361c1b14366ce36118e82b90faf7c7b1b136
+build_bzlib()
+{
+	echo "build_bzlib()
+################################################################################
+################################################################################
+################################################################################
+"
+
+	B_dir="bzip2-${BZIP2_SOURCE_VERSION}"
+	rm -rf $B_dir
+
+	verified_download $B_dir.tar.gz $BZIP2_SOURCE_SHA256 \
+		http://distfiles.gentoo.org/distfiles/bzip2-${BZIP2_SOURCE_VERSION}.tar.gz
+
+	tar -xf $B_dir.tar.gz
+	cd $B_dir
+	sed -i "/^CC=.*/d" Makefile
+	sed -i "/^AR=.*/d" Makefile
+	sed -i "/^RANLIB=.*/d" Makefile
+	sed -i "/^LDFLAGS=.*/d" Makefile
+	sed -i "s/^all: libbz2.a bzip2 bzip2recover test/all: libbz2.a bzip2 bzip2recover/" Makefile
+	make -j${HOST_NUM_CPU}
+	make install PREFIX=${PREFIX}
+#	sed -i "/^CC=.*/d" Makefile-libbz2_so
+#	make -f Makefile-libbz2_so -j${HOST_NUM_CPU}
+#	cp libbz2.so.1.0.6 ${SYSROOT}/usr/lib/libbz2.so
+	cd ..
+}
+
+## More information available at http://doc.qt.io/qt-5/opensslsupport.html
+build_openssl()
+{
+	echo "build_openssl()
+################################################################################
+################################################################################
+################################################################################
+"
+
+	B_dir="openssl-${OPENSSL_SOURCE_VERSION}"
+	rm -rf $B_dir
+
+	verified_download $B_dir.tar.gz $OPENSSL_SOURCE_SHA256 \
+		https://www.openssl.org/source/$B_dir.tar.gz
+
+	tar -xf $B_dir.tar.gz
+	cd $B_dir
+## We link openssl statically to avoid android silently sneaking in his own
+## version of libssl.so (we noticed this because it had some missing symbol
+## that made RS crash), the crash in some android version is only one of the
+## possible problems the fact that android insert his own binary libssl.so pose
+## non neglegible security concerns.
+	oBits="32"
+	[[ ${ANDROID_NDK_ARCH} =~ .*64.* ]] && oBits=64
+
+	ANDROID_NDK="${ANDROID_NDK_PATH}" PATH="${SYSROOT}/bin/:${PATH}" \
+		./Configure linux-generic${oBits} -fPIC --prefix="${PREFIX}" \
+		--openssldir="${SYSROOT}/etc/ssl"
+#	sed -i 's/LIBNAME=$$i LIBVERSION=$(SHLIB_MAJOR).$(SHLIB_MINOR) \\/LIBNAME=$$i \\/g' Makefile
+#	sed -i '/LIBCOMPATVERSIONS=";$(SHLIB_VERSION_HISTORY)" \\/d' Makefile
+	make -j${HOST_NUM_CPU}
+	make install
+	rm -f ${PREFIX}/lib/libssl.so*
+	rm -f ${PREFIX}/lib/libcrypto.so*
+	cd ..
+}
+
+build_sqlite()
+{
+	echo "build_sqlite()
+################################################################################
+################################################################################
+################################################################################
+"
+
+	B_dir="sqlite-autoconf-${SQLITE_SOURCE_VERSION}"
+	rm -rf $B_dir
+
+	verified_download $B_dir.tar.gz $SQLITE_SOURCE_SHA256 \
+		https://www.sqlite.org/${SQLITE_SOURCE_YEAR}/$B_dir.tar.gz
+
+	tar -xf $B_dir.tar.gz
+	cd $B_dir
+	./configure --with-pic --prefix="${PREFIX}" --host=${cArch}-linux
+	make -j${HOST_NUM_CPU}
+	make install
+	rm -f ${PREFIX}/lib/libsqlite3.so*
+#	${CC} -shared -o libsqlite3.so -fPIC sqlite3.o -ldl
+#	cp libsqlite3.so "${SYSROOT}/usr/lib"
+	cd ..
+}
+
+build_sqlcipher()
+{
+	echo "build_sqlcipher()
+################################################################################
+################################################################################
+################################################################################
+"
+
+	B_dir="sqlcipher-${SQLCIPHER_SOURCE_VERSION}"
+	rm -rf $B_dir
+
+	T_file="${B_dir}.tar.gz"
+
+	verified_download $T_file $SQLCIPHER_SOURCE_SHA256 \
+		https://github.com/sqlcipher/sqlcipher/archive/v${SQLCIPHER_SOURCE_VERSION}.tar.gz
+
+	tar -xf $T_file
+	cd $B_dir
+	case "${ANDROID_NDK_ARCH}" in
+	"arm64")
+	# SQLCipher config.sub is outdated and doesn't recognize newer architectures
+		rm config.sub
+		autoreconf --verbose --install --force
+		automake --add-missing --copy --force-missing
+	;;
+	esac
+	./configure --with-pic --build=$(sh ./config.guess) \
+		--host=${cArch}-linux \
+		--prefix="${PREFIX}" --with-sysroot="${SYSROOT}" \
+		--enable-tempstore=yes \
+		--disable-tcl --disable-shared \
+		CFLAGS="-DSQLITE_HAS_CODEC" LDFLAGS="${PREFIX}/lib/libcrypto.a"
+	make -j${HOST_NUM_CPU}
+	make install
+	cd ..
+}
+
+build_libupnp()
+{
+	echo "build_libupnp()
+################################################################################
+################################################################################
+################################################################################
+"
+
+	B_dir="pupnp-release-${LIBUPNP_SOURCE_VERSION}"
+	B_ext=".tar.gz"
+	B_file="${B_dir}${B_ext}"
+	rm -rf $B_dir
+
+	verified_download $B_file $LIBUPNP_SOURCE_SHA256 \
+		https://github.com/mrjimenez/pupnp/archive/release-${LIBUPNP_SOURCE_VERSION}${B_ext}
+
+	tar -xf $B_file
+	cd $B_dir
+	./bootstrap
+## liupnp must be configured as static library because if not the linker will
+## look for libthreadutils.so.6 at runtime that cannot be packaged on android
+## as it supports only libname.so format for libraries, thus resulting in a
+## crash at startup.
+	./configure --with-pic --enable-static --disable-shared --disable-samples \
+		--disable-largefile \
+		--prefix="${PREFIX}" --host=${cArch}-linux
+	make -j${HOST_NUM_CPU}
+	make install
+	cd ..
+}
+
+build_rapidjson()
+{
+	echo "build_rapidjson()
+################################################################################
+################################################################################
+################################################################################
+"
+
+	B_dir="rapidjson-${RAPIDJSON_SOURCE_VERSION}"
+	D_file="${B_dir}.tar.gz"
+	verified_download $D_file $RAPIDJSON_SOURCE_SHA256 \
+		https://github.com/Tencent/rapidjson/archive/v${RAPIDJSON_SOURCE_VERSION}.tar.gz
+	tar -xf $D_file
+	cp -r "${B_dir}/include/rapidjson/" "${PREFIX}/include/rapidjson"
+}
+
+build_restbed()
+{
+	echo "build_restbed()
+################################################################################
+################################################################################
+################################################################################
+"
+
+	[ -d restbed ] || git clone --depth=2000 https://github.com/Corvusoft/restbed.git
+	cd restbed
+#	git fetch --tags
+#	git checkout tags/${RESTBED_SOURCE_VERSION}
+	git checkout ${RESTBED_SOURCE_VERSION}
+	git submodule update --init dependency/asio
+	git submodule update --init dependency/catch
+	git submodule update --init dependency/kashmir
+	cd ..
+
+	rm -rf restbed-build; mkdir restbed-build ; cd restbed-build
+	cmake \
+		-DCMAKE_POSITION_INDEPENDENT_CODE=ON BUILD_TESTS=OFF \
+		-DBUILD_SSL=OFF -DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -H../restbed
+	make -j${HOST_NUM_CPU}
+	make install
+	cp "${PREFIX}/library/librestbed.a" "${PREFIX}/lib/"
+	cd ..
+}
+
+build_udp-discovery-cpp()
+{
+	echo "build_udp-discovery-cpp()
+################################################################################
+################################################################################
+################################################################################
+"
+
+	S_dir="udp-discovery-cpp"
+	[ -d $S_dir ] || git clone $UDP_DISCOVERY_CPP_SOURCE $S_dir
+	cd $S_dir
+	git checkout $UDP_DISCOVERY_CPP_VERSION
+	cd ..
+
+	B_dir="udp-discovery-cpp-build"
+	rm -rf ${B_dir}; mkdir ${B_dir}; cd ${B_dir}
+	cmake \
+		-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+		-DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -H../udp-discovery-cpp
+	make -j${HOST_NUM_CPU}
+	cp libudp-discovery.a "${PREFIX}/lib/"
+	cp ../$S_dir/*.hpp "${PREFIX}/include/"
+	cd ..
+}
+
+build_xapian()
+{
+	echo "build_xapian()
+################################################################################
+################################################################################
+################################################################################
+"
+
+	B_dir="xapian-core-${XAPIAN_SOURCE_VERSION}"
+	D_file="$B_dir.tar.xz"
+	verified_download $D_file $XAPIAN_SOURCE_SHA256 \
+		https://oligarchy.co.uk/xapian/${XAPIAN_SOURCE_VERSION}/$D_file
+	rm -rf $B_dir
+	tar -xf $D_file
+	cd $B_dir
+	B_endiannes_detection_failure_workaround="ac_cv_c_bigendian=no"
+	B_large_file=""
+	[ "${ANDROID_PLATFORM_VER}" -lt "24" ] && B_large_file="--disable-largefile"
+	./configure ${B_endiannes_detection_failure_workaround} ${B_large_file} \
+		--with-pic \
+		--disable-backend-inmemory --disable-backend-remote \
+		--disable--backend-chert --enable-backend-glass \
+		--host=${cArch}-linux --enable-static --disable-shared \
+		--prefix="${PREFIX}" --with-sysroot="${SYSROOT}"
+	make -j${HOST_NUM_CPU}
+	make install
+}
+
+build_miniupnpc()
+{
+	echo "build_miniupnpc()
+################################################################################
+################################################################################
+################################################################################
+"
+	S_dir="miniupnpc-${MINIUPNPC_SOURCE_VERSION}"
+	B_dir="miniupnpc-${MINIUPNPC_SOURCE_VERSION}-build"
+	D_file="$S_dir.tar.gz"
+	verified_download $D_file $MINIUPNPC_SOURCE_SHA256 \
+		http://miniupnp.free.fr/files/${D_file}
+	rm -rf $S_dir $B_dir
+	tar -xf $D_file
+	mkdir $B_dir
+	cd $B_dir
+	cmake \
+		-DUPNPC_BUILD_STATIC=TRUE \
+		-DUPNPC_BUILD_SHARED=FALSE \
+		-DUPNPC_BUILD_TESTS=FALSE \
+		-DUPNPC_BUILD_SAMPLE=FALSE \
+		-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+		-DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -S../$S_dir
+	make -j${HOST_NUM_CPU}
+	make install
+}
+
+build_toolchain
+[ "${INSTALL_QT_ANDROID}X" != "trueX" ] || install_qt_android
+build_bzlib
+build_openssl
+build_sqlite
+build_sqlcipher
+build_libupnp
+build_rapidjson
+build_restbed
+build_udp-discovery-cpp
+build_xapian
+build_miniupnpc
+delete_copied_includes
+
+echo NATIVE_LIBS_TOOLCHAIN_PATH=${NATIVE_LIBS_TOOLCHAIN_PATH}

build_scripts/Android/pull_sysroot.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Script to pull debugging sysroot from Android devices
+#
+# Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>
+# Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation, version 3.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>
+
+<<"ASCIIDOC"
+
+Pull files from Android device to prepare a debugging sysroot
+Inspired by:
+https://fw4spl-org.github.io/fw4spl-blog/2015/07/27/Native-debugging-on-Android-with-QtCreator.html
+
+The goal is to have a local copy of the sysroot of the connected device.
+For that we use the command `adb pull <remote_dir_or_file> <local_dir_or_file>`
+
+We will get a copy of:
+
+- `/system/lib/`
+- `/vendor/lib/`
+- `libc.so`
+- `libcutils.so`
+
+As well as the binaries `linker` and `app_process`
+
+IMPORTANT:
+from one device to another, the remote file `app_process` can be a binary file
+or a symlink to a binary file - which is NOT ok for us.
+That's so we will try to pull every known possible variants of `app_process`,
+e.g. `app_process32` or `app_process_init`
+
+ASCIIDOC
+
+
+## Define default value for variable, take two arguments, $1 variable name,
+## $2 default variable value, if the variable is not already define define it
+## with default value.
+function define_default_value()
+{
+	VAR_NAME="${1}"
+	DEFAULT_VALUE="${2}"
+
+	[ -z "${!VAR_NAME}" ] && export ${VAR_NAME}="${DEFAULT_VALUE}"
+}
+
+define_default_value ANDROID_SERIAL "$(adb devices | head -n 2 | tail -n 1 | awk '{print $1}')"
+define_default_value DEBUG_SYSROOT "${HOME}/Builds/debug_sysroot/${ANDROID_SERIAL}/"
+
+rm -rf "${DEBUG_SYSROOT}"
+
+for mDir in  "/system/lib/" "/vendor/lib/"; do
+	mkdir -p "${DEBUG_SYSROOT}/${mDir}"
+	# adb pull doesn't behave like rsync dirA/ dirB/ so avoid nesting the
+	# directory by deleting last one before copying
+	rmdir "${DEBUG_SYSROOT}/${mDir}"
+	adb pull "${mDir}" "${DEBUG_SYSROOT}/${mDir}"
+done
+
+# Retrieve the specific binaries - some of these adb commands will fail, since
+# some files may not exist, but that's ok.
+
+mkdir -p "${DEBUG_SYSROOT}/system/bin/"
+for mBin in "/system/bin/linker" "/system/bin/app_process" \
+	"/system/bin/app_process_init" "/system/bin/app_process32" \
+	"/system/bin/app_process64" ; do
+	adb pull "${mBin}" "${DEBUG_SYSROOT}/${mBin}"
+done
+
+# Verify which variants of the specific binaries could be pulled 
+echo
+echo "Found the following specific binaries:"
+echo
+ls -1 "${DEBUG_SYSROOT}/system/bin/"*
+ls -1 "${DEBUG_SYSROOT}/system/lib/libc.so"*
+ls -1 "${DEBUG_SYSROOT}/system/lib/libcutils.so"*
+echo
+
+echo DEBUG_SYSROOT="${DEBUG_SYSROOT}"

build_scripts/Android/start_gdbserver.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+
+# Script to start gdbserver on Android device and attach to retroshare-service
+#
+# Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>
+# Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation, version 3.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>
+
+<<"ASCIIDOC"
+
+Start gdbserver on Android device and attach it to the retroshare service
+process
+
+Inspired by:
+https://fw4spl-org.github.io/fw4spl-blog/2015/07/27/Native-debugging-on-Android-with-QtCreator.html
+
+ASCIIDOC
+
+
+## Define default value for variable, take two arguments, $1 variable name,
+## $2 default variable value, if the variable is not already define define it
+## with default value.
+function define_default_value()
+{
+	VAR_NAME="${1}"
+	DEFAULT_VALUE="${2}"
+
+	[ -z "${!VAR_NAME}" ] && export ${VAR_NAME}="${DEFAULT_VALUE}"
+}
+
+define_default_value ANDROID_APK_PACKAGE "org.retroshare.service"
+define_default_value ANDROID_PROCESS_NAME "org.retroshare.service:rs"
+define_default_value ANDROID_INSTALL_PATH ""
+define_default_value LIB_GDB_SERVER_PATH ""
+define_default_value ANDROID_SERIAL "$(adb devices | head -n 2 | tail -n 1 | awk '{print $1}')"
+define_default_value GDB_SERVER_PORT 5039
+
+adb_ushell()
+{
+	adb shell run-as ${ANDROID_APK_PACKAGE} $@
+}
+
+[ -z "${ANDROID_INSTALL_PATH}" ] &&
+{
+	ANDROID_INSTALL_PATH="$(adb_ushell pm path "${ANDROID_APK_PACKAGE}")"
+	ANDROID_INSTALL_PATH="$(dirname ${ANDROID_INSTALL_PATH#"package:"})"
+	[ -z "${ANDROID_INSTALL_PATH}" ] &&
+		cat <<EOF
+Cannot find install path for ${ANDROID_APK_PACKAGE} make sure it is installed,
+or manually specify ANDROID_INSTALL_PATH
+
+
+EOF
+}
+
+## If not passed as environement variable try to determine gdbserver path
+## shipped withing APK
+[ -z "${LIB_GDB_SERVER_PATH}" ] &&
+{
+	
+	for mUsualPath in \
+		"${ANDROID_INSTALL_PATH}/lib/libgdbserver.so" \
+		"${ANDROID_INSTALL_PATH}/lib/arm64/libgdbserver.so"
+	do
+		adb_ushell ls "${mUsualPath}" &&
+			export LIB_GDB_SERVER_PATH="${mUsualPath}" && break
+	done
+}
+
+[ -z "${LIB_GDB_SERVER_PATH}" ] &&
+{
+	cat <<EOF
+libgdbserver.so not found in any of the usual path attempting to look for it
+with find, it will take a little more time. Take note of the discovered path and
+define LIB_GDB_SERVER_PATH on your commandline at next run to avoid waiting
+again.
+
+
+EOF
+	tFile="$(mktemp)"
+	adb_ushell find ${ANDROID_INSTALL_PATH} -type f -name 'libgdbserver.so' | \
+		tee "${tFile}"
+
+	LIB_GDB_SERVER_PATH="$(head -n 1 "${tFile}")"
+	rm "${tFile}"
+}
+
+[ -z "${LIB_GDB_SERVER_PATH}" ] &&
+{
+	echo "Cannot find libgdbserver.so, are you sure your package ships it?"
+	exit -1
+}
+
+mPid="$(adb_ushell ps | grep ${ANDROID_PROCESS_NAME} | awk '{print $2}')"
+[ -z "${mPid}" ] &&
+{
+	echo "Failed ${ANDROID_PROCESS_NAME} PID retrival are you sure it is running?"
+	exit -2
+}
+
+
+## Establish port forwarding so we can connect to gdbserver with gdb
+adb forward tcp:${GDB_SERVER_PORT} tcp:${GDB_SERVER_PORT}
+
+((adb_ushell ${LIB_GDB_SERVER_PATH} 127.0.0.1:${GDB_SERVER_PORT} --attach ${mPid})&)

build_scripts/GitlabCI/Android.Dockerfile

@@ -1,48 +0,0 @@
-ARG ANDROID_NDK_ARCH=arm
-FROM registry.gitlab.com/retroshare/retroshare:android_${ANDROID_NDK_ARCH}_base
-
-ENV APT_UNAT="--assume-yes --quiet"
-
-RUN apt-get update $APT_UNAT && apt-get upgrade $APT_UNAT --show-upgraded
-
-ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
-ARG REPO_BRANCH=master
-
-WORKDIR /RetroShare
-RUN git remote add testing $REPO_URL && \
-	git fetch --tags testing $REPO_BRANCH && \
-	git reset --hard testing/$REPO_BRANCH && \
-	git --no-pager log --max-count 1
-
-RUN rm -rf /jsonapi-generator-build ; mkdir /jsonapi-generator-build
-WORKDIR /jsonapi-generator-build/
-RUN qmake ../RetroShare/jsonapi-generator/src/ \
-		CONFIG+=no_retroshare_plugins \
-		CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
-		CONFIG+=rs_jsonapi CONFIG+=no_rs_sam3_libsam3 && \
-	make -j$(nproc)
-
-RUN rm -rf /retroshare-service-android-build ; mkdir /retroshare-service-android-build
-WORKDIR /retroshare-service-android-build
-
-# ARG declared before FROM get wiped after it, so we need declaring it again
-ARG ANDROID_NDK_ARCH=arm
-ARG RS_SERVICE_QMAKE_EXTRA_OPTS
-RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
-	-spec android-clang \
-	CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
-	RS_UPNP_LIB=miniupnpc \
-	JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
-	NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
-	CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
-	CONFIG+=no_rs_service_terminal_login \
-	CONFIG+=no_rs_sam3 CONFIG+=no_rs_sam3_libsam3 \
-	$RS_SERVICE_QMAKE_EXTRA_OPTS
-RUN	make -j$(nproc)
-RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/
-
-ARG ANDROID_PLATFORM_VER=16
-RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/androiddeployqt \
-	--input retroshare-service/src/android-libretroshare-service.so-deployment-settings.json \
-	--output android-build --android-platform android-$ANDROID_PLATFORM_VER \
-	--jdk $JAVA_HOME --gradle

build_scripts/GitlabCI/base.Dockerfile

@@ -2,61 +2,42 @@
 ## force cloning a new
 
 ## To prepare an image suitable as base for Gitlab CI use
-# export CI_REGISTRY_IMAGE="registry.gitlab.com/retroshare/retroshare:base"
-# docker build -t "${CI_REGISTRY_IMAGE}" -f base.Dockerfile .
+# docker build -t "${CI_REGISTRY_IMAGE}:base" --build-arg KEEP_SOURCE=true --build-arg REPO_DEPTH="" -f base.Dockerfile .
+
+## Now you need to tag it so you can later push it
+# docker tag ${ID_OF_THE_CREATED_IMAGE} registry.gitlab.com/retroshare/${CI_REGISTRY_IMAGE}:base
 
 ## To push it to gitlab CI registry you need first to login and the to push
 # docker login registry.gitlab.com
-# docker push "${CI_REGISTRY_IMAGE}"
+# docker push registry.gitlab.com/retroshare/${CI_REGISTRY_IMAGE}:base
+
 
 ## To run the container
-# docker run -it -p 127.0.0.1:9092:9092 "${CI_REGISTRY_IMAGE}" retroshare-service --jsonApiPort 9092 --jsonApiBindAddress 0.0.0.0
+# docker run -it -p 127.0.0.1:9092:9092 "${CI_REGISTRY_IMAGE}:base" retroshare-service --jsonApiPort 9092 --jsonApiBindAddress 0.0.0.0
 
-FROM ubuntu:22.04
+FROM ubuntu
 
-ENV DEBIAN_FRONTEND=noninteractive
-ENV APT_UNAT="--assume-yes --quiet"
-
-RUN apt-get update $APT_UNAT && \
-	apt-get upgrade --show-upgraded $APT_UNAT && \
-	apt-get clean $APT_UNAT && \
-	apt-get install --no-install-recommends $APT_UNAT \
-		bash build-essential cimg-dev libssl-dev libbz2-dev \
-		libminiupnpc-dev \
-		libsqlite3-dev libsqlcipher-dev \
-		pkg-config libz-dev \
-		libxapian-dev doxygen rapidjson-dev \
-		git cmake curl python3
-
-## Avoid git cloning spuriously failing with
-#  server certificate verification failed. CAfile: none CRLfile: none
-RUN apt-get install --no-install-recommends $APT_UNAT --reinstall \
-	ca-certificates
-
-RUN git clone --depth 1 https://github.com/aetilius/pHash.git && \
-	rm -rf pHash-build && mkdir pHash-build && cd pHash-build && \
-	cmake -B. -H../pHash -DCMAKE_INSTALL_PREFIX=/usr && \
-	make -j$(nproc) && make install && cd .. && \
-	rm -rf pHash-build pHash
+ARG CACHEBUST=0
+RUN \
+	apt-get update -y && apt-get upgrade -y && \
+	apt-get install -y build-essential libssl-dev libbz2-dev libsqlite3-dev \
+		libsqlcipher-dev libupnp-dev pkg-config libz-dev \
+		qt5-default libxapian-dev qttools5-dev doxygen rapidjson-dev \
+		git cmake curl
 
 ARG FRESHCLONE=0
-ARG REPO_URL=https://github.com/RetroShare/RetroShare.git
+ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
 ARG REPO_BRANCH=master
 ARG REPO_DEPTH="--depth 2000"
-RUN git clone $REPO_DEPTH $REPO_URL -b $REPO_BRANCH && \
-	cd RetroShare && \
-	git fetch --tags && \
-	git submodule update --init \
-		libbitdht/ libretroshare/ openpgpsdk/ retroshare-webui/ \
-		supportlibs/restbed/ && \
-	cd supportlibs/restbed/ && \
-	git submodule update --init \
-		dependency/asio/ dependency/kashmir/ && \
-	cd ../../../
-
+ARG KEEP_SOURCE=false
+RUN apt-get update -y && apt-get upgrade -y
+RUN git clone $REPO_DEPTH $REPO_URL -b $REPO_BRANCH && cd RetroShare && \
+	git fetch --tags && cd ..
 RUN \
 	mkdir RetroShare-build && cd RetroShare-build && \
-	cmake -B. -S../RetroShare/retroshare-service \
-		-DRS_FORUM_DEEP_INDEX=ON -DRS_JSON_API=ON -DRS_WEBUI=ON && \
-	make -j$(nproc) && make install && \
-	cd .. && rm -rf RetroShare-build
+	qmake ../RetroShare \
+		CONFIG+=no_retroshare_plugins CONFIG+=ipv6 \
+		CONFIG+=retroshare_service CONFIG+=no_retroshare_gui \
+		CONFIG+=rs_jsonapi CONFIG+=rs_deep_search && \
+	(make -j$(nproc) || make -j$(nproc) || make) && make install && \
+	cd .. && rm -rf RetroShare-build && ($KEEP_SOURCE || rm -rf RetroShare)

build_scripts/GitlabCI/getRepoArgs.sh

@@ -1,9 +0,0 @@
-#!/bin/sh
-
-[ -n "$CI_MERGE_REQUEST_ID" ] &&
-	echo \
-		--build-arg REPO_URL="$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" \
-		--build-arg REPO_BRANCH="$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ||
-	echo \
-		--build-arg REPO_URL="$CI_REPOSITORY_URL" \
-		--build-arg REPO_BRANCH="$CI_COMMIT_BRANCH"

build_scripts/GitlabCI/gitlabCI.Dockerfile

@@ -2,19 +2,17 @@ FROM registry.gitlab.com/retroshare/retroshare:base
 
 RUN apt-get update -y && apt-get upgrade -y
 
-ARG REPO_URL=https://github.com/RetroShare/RetroShare.git
+ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
 ARG REPO_BRANCH=master
 RUN \
 	cd RetroShare && git remote add testing $REPO_URL && \
 	git fetch --tags testing $REPO_BRANCH && \
 	git reset --hard testing/$REPO_BRANCH && \
-	git submodule update --init \
-		libbitdht/ libretroshare/ openpgpsdk/ retroshare-webui/ && \
 	git --no-pager log --max-count 1
 RUN \
 	mkdir RetroShare-build && cd RetroShare-build && \
-	cmake -B. -S../RetroShare/retroshare-service \
-		-DRS_FORUM_DEEP_INDEX=ON -DRS_JSON_API=ON -DRS_WEBUI=ON \
-		-DRS_WARN_DEPRECATED=OFF -DRS_WARN_LESS=ON && \
-	make -j$(nproc) && make install && \
+	qmake ../RetroShare CONFIG+=no_retroshare_gui \
+		CONFIG+=retroshare_service \
+		CONFIG+=rs_jsonapi CONFIG+=rs_deep_search && \
+	(make -j$(nproc) || make -j$(nproc) || make) && make install && \
 	cd .. && rm -rf RetroShare-build

build_scripts/OBS

@@ -1 +1 @@
-Subproject commit 0a3997cc1355b2c848161dca015b7e2df039707b
+Subproject commit b0d7ae39fe9d9192848bbf7b8902d2188da895c9

build_scripts/OSX/MacOS.10.6_RS_Compilation_Instructions.txt

@@ -0,0 +1,60 @@
+Hi there,
+
+I heard about RetroShare recently (on Slashdot) and really like the goal of the program; it's something I've been wanting for a long time now. I'm a software developer so I figured I could help contribute to the project. I started by getting a build up and running on my mac, which was non-trivial as it looks like the Mac build hasn't been maintained (or maybe it's just targetting older OS X version?). Anyway, I have instructions and patches to get a build going on OS X 10.6, if you want to put them up on the wiki or something. The steps I used boiled down to running the following commands in a directory that also contains the retroshare-mac-build.patch file, which I posted at https://staktrace.com/pub/retroshare-mac-build.patch (you can also see the changes at https://github.com/staktrace/retroshare/commit/51f554f909086f4baca7be215d5edacab744dea4)
+
+sudo port selfupdate
+sudo port install qt4-mac
+sudo port install wget
+
+wget ftp://ftp.gnupg.org/gcrypt/libgpg-error/libgpg-error-1.10.tar.bz2
+tar xjf libgpg-error-1.10.tar.bz2
+rm libgpg-error-1.10.tar.bz2
+pushd libgpg-error-1.10
+./configure --prefix=$PWD/build --enable-static=yes --enable-shared=no
+make
+make install
+popd
+
+wget ftp://ftp.gnupg.org/gcrypt/gpgme/gpgme-1.3.1.tar.bz2
+tar xjf gpgme-1.3.1.tar.bz2
+rm gpgme-1.3.1.tar.bz2
+pushd gpgme-1.3.1
+./configure --prefix=$PWD/build --enable-static=yes --enable-shared=no --with-gpg-error-prefix=$PWD/../libgpg-error-1.10/build
+make
+make install
+popd
+
+wget http://miniupnp.free.fr/files/download.php?file=miniupnpc-1.3.tar.gz
+tar xzf miniupnpc-1.3.tar.gz
+rm miniupnpc-1.3.tar.gz
+pushd miniupnpc-1.3
+make upnpc-static
+popd
+
+git clone https://github.com/kigeia/retroshare
+pushd retroshare
+git apply ../retroshare-mac-build.patch
+popd
+
+pushd retroshare/libbitdht/src
+qmake
+make
+popd
+
+pushd retroshare/libretroshare/src
+qmake
+make
+popd
+
+pushd retroshare/retroshare-gui/src
+qmake
+make
+popd
+
+At the end there is a RetroShare.App in the retroshare/retroshare-gui/ folder which seems to work as expected.
+
+Now that I've gotten it building and working, I'd like to start working on adding features. One that I would like to see is taking advantage of the RetroShare platform to enable F2F games. A while back I wrote a P2P collaborative crossword solver app in Java; porting that to work as a RetroShare plugin would probably be a good start for me. I looked briefly at the existing plugins in the source tree but haven't yet had time to peruse the rsplugin.h API in detail; if you have any tips or pointers before I dive in, please do let me know. I'll probably start work on it in a couple of days and progress might be a little slow because I'm also fairly busy with other things right now.
+
+Cheers,
+kats
+

build_scripts/OSX/MacOS_X_InstallGuide.md

@@ -8,44 +8,35 @@ Use default options. And add Qt Script support.
 
 Add to the PATH environment variable by editing your *~/.profile* file.
 
-       export PATH="/users/$USER/Qt/5.14.1/clang_64/bin:$PATH"
+       export PATH="/users/$USER/Qt/5.5/clang_64/bin:$PATH"
 
 Depends on which version of Qt you use.
 
-## Get RetroShare
+## ***Choose if you use MacPort or HomeBrew***
 
-In Qt Creator Projects -> New -> Import Project -> Git Clone -> Choose
-Add Repository and Continoue
+### MacPort Installation
 
-       Repository: https://github.com/RetroShare/RetroShare.git 
+Install MacPort and XCode following this guide: [MacPort and XCode](http://guide.macports.org/#installing.xcode)
 
-via Terminal:
+Start XCode to get it updated and to able C compiler to create executables.
 
-       cd <your development directory>
-       git clone https://github.com/RetroShare/RetroShare.git retroshare
+#### Install libraries  
 
-via GitHub Desktop: [GitHub Desktop Download](https://central.github.com/deployments/desktop/desktop/latest/darwin)
+       $sudo port -v selfupdate
+       $sudo port install openssl
+       $sudo port install miniupnpc
+       $sudo port install libmicrohttpd
+       
+For VOIP Plugin: 
 
-In GitHub Desktop -> Clone Repository -> URL
+       $sudo port install speex-devel
+       $sudo port install opencv
 
-       Add Repository URL: https://github.com/RetroShare/RetroShare.git and Clone
+Get Your OSX SDK if missing: [MacOSX-SDKs](https://github.com/phracker/MacOSX-SDKs)
 
-## ***Get XCode & MacOSX SDK***
+### HOMEBREW Installation
 
-Install XCode following this guide: [XCode](http://guide.macports.org/#installing.xcode)
-
-To identify the correct version of Xcode to install, you need to know which OS you are running. Go to the [x] menu -> "About This Mac" and read the macOS version number.
-
-If you are running the macOS Catalina >= 10.15, you can install Xcode directly from App Store using the instructions below. 
-
-You can find older versions of Xcode at [Apple Developer Downloads](https://developer.apple.com/downloads/). Find the appropriate .xip file for your macOS version
-
-To install from App Store:
-
-Select [x] menu - > "App Store…".
-Search for Xcode. Download and install.
-
-Once Xcode has installed, you must drag the XCode icon into your Applications folder. After you have done this, open Xcode from the Applications folder by double-clicking on the icon and then follow the remaining instructions below. 
+Install HomeBrew following this guide: [HomeBrew](http://brew.sh/)
 
 Install XCode command line developer tools:
 
@@ -53,48 +44,11 @@ Install XCode command line developer tools:
 
 Start XCode to get it updated and to able C compiler to create executables.
 
-Get Your MacOSX SDK if missing: [MacOSX-SDKs](https://github.com/phracker/MacOSX-SDKs)
-
-## ***Choose if you use MacPort or HomeBrew***
-
-### MacPort Installation
-
-Install MacPort following this guide: [MacPort](http://guide.macports.org/#installing.xcode)
-
 #### Install libraries  
 
-       $ sudo port -v selfupdate
-       $ sudo port install openssl
-       $ sudo port install miniupnpc
-       
-For VOIP Plugin: 
-
-       $ sudo port install speex-devel
-       $ sudo port install opencv
-       $ sudo port install ffmpeg
-
-
-### HOMEBREW Installation
-
-Install HomeBrew following this guide: [HomeBrew](http://brew.sh/)
-
-#### Install libraries  
-
-       $ brew install openssl
-       $ brew install miniupnpc
-       $ brew install rapidjson
-       $ brew install sqlcipher
-
-For RNP lib:
-
-	   $ brew install bzip2
-	   $ brew install zlib
-	   $ brew install json-c
-	   $ brew install botan@2
-
-#### Install CMake 
-
-       $ brew install cmake
+       $brew install openssl
+       $brew install miniupnpc
+       $brew install libmicrohttpd
        
 If you have error in linking, run this:
 
@@ -102,130 +56,37 @@ If you have error in linking, run this:
 
 For VOIP Plugin: 
 
-       $ brew install speex
-       $ brew install speexdsp
-       $ brew install opencv
-       $ brew install ffmpeg
+       $brew install speex
+       $brew install speexdsp
+       $brew install homebrew/science/opencv
+       $brew install ffmpeg
 
-For FeedReader Plugin:
-
-       $ brew install libxslt
-       $ brew install libxml2
+Get Your OSX SDK if missing: [MacOSX-SDKs](https://github.com/phracker/MacOSX-SDKs)
 
 ## Last Settings
 
-In QtCreator Projects -> Manage Kits > Version Control > Git:
+In QtCreator Option Git select "Pull" with "Rebase"
 
-       select "Pull with rebase"
+## Compil missing libraries
+### SQLCipher
+       
+       cd <your development directory>
+       git clone https://github.com/sqlcipher/sqlcipher.git
+       cd sqlcipher
+       ./configure --disable-shared --disable-tcl --enable-static --enable-tempstore=yes CFLAGS="-DSQLITE_HAS_CODEC -I/opt/local/include" LDFLAGS="-lcrypto"
+       make 
+       sudo make install
 
-In QtCreator Projects -> Build -> Build Settings -> Build Environment -> Add this path:
+NOTE, might be necessary to *chmod 000 /usr/local/ssl* temporarily during *./configure* if 
+homebrew uses newer, non-stock ssl dependencies found there. Configure might get confused.
 
-       /usr/local/bin
+You can now compile RS into Qt Creator or with terminal
 
-In QtCreator Projects -> Build -> Build Settings -> Build Steps -> Add Additional arguments:
-
-       "CONFIG+=rs_autologin" "CONFIG+=rs_use_native_dialogs" 
-
-## Set your Mac OS SDK version
-
-
-Edit retroshare.pri and set your installed sdk version example for 11.1 -> rs_macos11.1 (line 135:)
-
-    macx:CONFIG *= rs_macos11.1
-    rs_macos10.8:CONFIG -= rs_macos11.1
-    rs_macos10.9:CONFIG -= rs_macos11.1
-    rs_macos10.10:CONFIG -= rs_macos11.1
-    rs_macos10.12:CONFIG -= rs_macos11.1
-    rs_macos10.13:CONFIG -= rs_macos11.1
-    rs_macos10.14:CONFIG -= rs_macos11.1
-    rs_macos10.15:CONFIG -= rs_macos11.1
-
-
-## Link Include & Libraries
-
-When required edit your retroshare.pri macx-* section, check if the Include and Lib path are correct (macx-* section)
-
-    INCLUDEPATH += "/usr/local/opt/openssl/include"
-    QMAKE_LIBDIR += "/usr/local/opt/openssl/lib"
-    QMAKE_LIBDIR += "/usr/local/opt/sqlcipher/lib"
-    QMAKE_LIBDIR += "/usr/local/opt/miniupnpc/lib"
-
-alternative via Terminal
-
-    $ qmake 
-    INCLUDEPATH+="/usr/local/opt/openssl/include" \
-    QMAKE_LIBDIR+="/usr/local/opt/openssl/lib" \
-    QMAKE_LIBDIR+="/usr/local/opt/sqlcipher/lib" \
-    QMAKE_LIBDIR+="/usr/local/opt/miniupnpc/lib" \
-    CONFIG+=rs_autologin \
-    CONFIG+=rs_use_native_dialogs \
-    CONFIG+=release \
-    ..
-
-For FeedReader Plugin:
-
-    INCLUDEPATH += "/usr/local/opt/libxml2/include/libxml2"
-
-With plugins:
-
-    $ qmake \
-    INCLUDEPATH+="/usr/local/opt/openssl/include" QMAKE_LIBDIR+="/usr/local/opt/openssl/lib" \
-    QMAKE_LIBDIR+="/usr/local/opt/sqlcipher/lib" \
-    QMAKE_LIBDIR+="/usr/local/opt/miniupnpc/lib" \
-    INCLUDEPATH+="/usr/local/opt/opencv/include/opencv4" QMAKE_LIBDIR+="/usr/local/opt/opencv/lib" \
-    INCLUDEPATH+="/usr/local/opt/speex/include" QMAKE_LIBDIR+="/usr/local/opt/speex/lib/" \
-    INCLUDEPATH+="/usr/local/opt/speexdsp/include" QMAKE_LIBDIR+="/usr/local/opt/speexdsp/lib/" \
-    INCLUDEPATH+="/usr/local/opt/libxslt/include" QMAKE_LIBDIR+="/usr/local/opt/libxslt/lib" \
-    QMAKE_LIBDIR+="/usr/local/opt/ffmpeg/lib" \
-    LIBS+=-lopencv_videoio \
-    CONFIG+=retroshare_plugins \
-    CONFIG+=rs_autologin \
-    CONFIG+=rs_use_native_dialogs \
-    CONFIG+=release \
-    ..
-
-## Compile RetroShare 
-
-You can now compile RetroShare into Qt Creator or with Terminal
-
-       $ cd /path/to/retroshare
-       $ qmake ..
-       $ make
+       cd <your development directory>
+       git clone https://github.com/RetroShare/RetroShare.git retroshare
+       cd retroshare
+       qmake; make
 
 You can change Target and SDK in *./retroshare.pri:82* changing value of QMAKE_MACOSX_DEPLOYMENT_TARGET and QMAKE_MAC_SDK
 
-You can find the compiled application at *./retroshare/retroshare-gui/src/retroshare.app*
-
-## Issues
-
-If you have issues with openssl (Undefined symbols for architecture x86_64) try to add to *~/.profile* file this or via Terminal
-
-       export PATH="/usr/local/opt/openssl/bin:$PATH"
-       export LDFLAGS="-L/usr/local/opt/openssl/lib"
-       export CPPFLAGS="-I/usr/local/opt/openssl/include"
-       export PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig"
-
-For Qt Creator -> QtCreator Projects -> Build -> Build Settings -> Build Steps -> Add Additional arguments:
-
-       LDFLAGS="-L/usr/local/opt/openssl/lib"
-       CPPFLAGS="-I/usr/local/opt/openssl/include"
-
-
-
-## Copy Plugins
-
-    $ cp \
-    ./plugins/FeedReader/lib/libFeedReader.dylib \
-    ./plugins/VOIP/lib/libVOIP.dylib \
-    ./plugins/RetroChess/lib/libRetroChess.dylib \
-    ./retroshare-gui/src/RetroShare.app/Contents/Resources/
-
-### Compile Retroshare-Service & Webui with CMake
-before you can compile overwrite the file "asio/include/asio/detail/config.hpp" here is a fix for macos [
-asio fix](https://github.com/chriskohlhoff/asio/commit/68df16d560c68944809bb2947360fe8035e9ae0a)   
-
-       $ cd retroshare-service
-       $ mkdir build-dir
-       $ cd build-dir
-       $ cmake -DRS_WEBUI=ON -DCMAKE_BUILD_TYPE=Release ..
-       $ make
+You can find compiled application on *./retroshare/retroshare-gui/src/retroshare.app*

build_scripts/OSX/OSX_RS_Compilation_Instructions.txt

@@ -0,0 +1,66 @@
+
+Mac OSX Build Instructions.
+-------------------------------------------
+
+There are several complications with building Retroshare under OSX.
+ 1) Support Libraries must be built and installed seperately.
+ 2) Universal and OSX 10.5 support is a little tricky, mainly due to the support libraries.
+
+Additional Libraries
+---------------------
+
+ * GnuPG Package, that comes with Retroshare OSX install image. (GnuPG-1.4.9.dmg)
+
+GPG Development libraries
+ * libassuan    (I'm using 2.0.1)
+ * libgpg-error (I'm using 1.9)
+ * libgpgme     (I'm using 1.3.0)
+
+These libraries use standard UNIX installation systems: AUTOCONF/AUTOMAKE (configure, make, etc)
+Unfortunately, this makes it difficult and a little manual to compile Universal and 10.5 libraries.
+
+UPNPC (for OSX and windows)
+ * miniupnpc    (I'm using 1.0)
+
+You will also need to install 
+ * XCode (available on the Apple Install CDs)
+ * Qt4 (from trolltech.com)
+
+First Compilation...
+-------------------- 
+
+The First challenge is to build Retroshare on your Mac. For this first compilation, 
+we only build for your specific machine, and not attempt a Generic / 10.5 / Univeral build.
+
+
+1) Install / Compile all the packages listed above.
+     be sure to use a configure command like this where applicable to only create a static library.
+    ./configure --enable-static=yes --enable-shared=no CFLAGS="-arch i386" CPPFLAGS="-arch i386"
+
+2) Check out the Retroshare SVN.
+
+3) compile libbitdht:
+	cd libbitdht/src
+	qmake    
+
+	This recreates a xcodeproj file for compilation using XCode.
+	Open with Xcode, and build.
+
+4) compile libretroshare: same way.
+5) compile retroshare-gui: same way.
+
+
+Creating Retroshare OSX Distribution Packages.
+-----------------------------------------------
+ TODO, once I've got feedback on First Compilation!
+
+
+
+
+	
+
+
+
+
+
+

build_scripts/OSX/makeOSXPackage.sh

@@ -1,51 +0,0 @@
-#!/bin/sh
-
-APP="RetroShare"
-RSVERSION="0.6.7a"
-QTVERSION="Qt-5.15.11"
-
-# Install the 7z to create dmg archives.
-#brew list p7zip || brew install p7zip
-
-# Package your app
-echo "Packaging retroshare..."
-#cd ${project_dir}/build/macOS/clang/x86_64/release/
-cd retroshare-gui/src/
-
-# Remove build directories that you don't want to deploy
-rm -rf moc
-rm -rf obj
-rm -rf qrc
-
-# This sets the CFBundleVersion & CFBundleShortVersionString string
-/usr/libexec/PlistBuddy -c "Delete :CFBundleGetInfoString" retroshare.app/Contents/Info.plist
-/usr/libexec/PlistBuddy -c "Add :CFBundleVersion string $RSVERSION" retroshare.app/Contents/Info.plist
-/usr/libexec/PlistBuddy -c "Add :CFBundleShortVersionString string $RSVERSION" retroshare.app/Contents/Info.plist
-/usr/libexec/PlistBuddy -c "Delete :NSRequiresAquaSystemAppearance" retroshare.app/Contents/Info.plist
-
-# This automatically creates retroshare.dmg
-
-echo "Creating dmg archive..."
-macdeployqt retroshare.app -dmg
-
-DATE=`date +"%m-%d-%Y"`
-MACVERSION=`sw_vers -productVersion`
-#RSVERSION=`git describe --abbrev=0 --tags`
-GITHEAD=`git rev-parse --short HEAD`
-
-mv $APP.dmg "$APP-$RSVERSION-$GITHEAD-$DATE-MacOS-$MACVERSION-$QTVERSION.dmg"
-
-# You can use the appdmg command line app to create your dmg file if
-# you want to use a custom background and icon arrangement. I'm still
-# working on this for my apps, myself. If you want to do this, you'll
-# remove the -dmg option above.
-# appdmg json-path YourApp_${TRAVIS_TAG}.dmg
-
-# Copy other project files
-cp "../../libbitdht/src/bitdht/bdboot.txt" "retroshare.app/Contents/Resources/"
-cp "../../plugins/FeedReader/lib/libFeedReader.dylib" "retroshare.app/Contents/Resources/"
-cp -R "sounds" "retroshare.app/Contents/Resources/sounds"
-
-# cp "${project_dir}/README.md" "README.md"
-# cp "${project_dir}/LICENSE" "LICENSE"
-# cp "${project_dir}/Qt License" "Qt License"

build_scripts/OSX/retroshare-mac-build.patch

@@ -0,0 +1,94 @@
+commit 51f554f909086f4baca7be215d5edacab744dea4
+Author: Kartikaya Gupta <kats@calvin.staktrace.com>
+Date:   Wed Mar 7 23:30:48 2012 -0500
+
+    Modifications needed to get build working on Mac OS X 10.6
+
+diff --git a/.gitignore b/.gitignore
+new file mode 100644
+index 0000000..3e90033
+--- /dev/null
++++ b/.gitignore
+@@ -0,0 +1,6 @@
++*.o
++Makefile
++libbitdht/src/lib/
++libretroshare/src/lib/
++retroshare-gui/src/RetroShare.app/
++retroshare-gui/src/temp/
+diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro
+index 6f35e01..267cc91 100644
+--- a/libretroshare/src/libretroshare.pro
++++ b/libretroshare/src/libretroshare.pro
+@@ -312,7 +312,7 @@ mac {
+ 		OBJECTS_DIR = temp/obj
+ 		MOC_DIR = temp/moc
+ 		#DEFINES = WINDOWS_SYS WIN32 STATICLIB MINGW
+-                #DEFINES *= MINIUPNPC_VERSION=13
++		DEFINES *= MINIUPNPC_VERSION=13
+ 		DESTDIR = lib
+ 
+                 #miniupnp implementation files
+@@ -326,12 +326,11 @@ mac {
+ 		# Beautiful Hack to fix 64bit file access.
+                 QMAKE_CXXFLAGS *= -Dfseeko64=fseeko -Dftello64=ftello -Dfopen64=fopen -Dvstatfs64=vstatfs
+ 
+-                UPNPC_DIR = ../../../miniupnpc-1.0
+-		GPG_ERROR_DIR = ../../../../libgpg-error-1.7
+-		GPGME_DIR  = ../../../../gpgme-1.1.8
++		UPNPC_DIR = ../../../miniupnpc-1.3
++		GPG_ERROR_DIR = ../../../libgpg-error-1.10
++		GPGME_DIR  = ../../../gpgme-1.3.1
+ 
+-		INCLUDEPATH += . $${UPNPC_DIR} 
+-		#INCLUDEPATH += . $${UPNPC_DIR} $${GPGME_DIR}/src $${GPG_ERROR_DIR}/src
++		INCLUDEPATH += . $${UPNPC_DIR} $${GPGME_DIR}/src $${GPG_ERROR_DIR}/src
+ }
+ 
+ ################################# FreeBSD ##########################################
+diff --git a/libretroshare/src/pqi/sslfns.cc b/libretroshare/src/pqi/sslfns.cc
+index 2588cb8..305433f 100644
+--- a/libretroshare/src/pqi/sslfns.cc
++++ b/libretroshare/src/pqi/sslfns.cc
+@@ -586,8 +586,8 @@ X509 *loadX509FromDER(const uint8_t *ptr, uint32_t len)
+ 	X509 *tmp = NULL;
+ #ifdef __APPLE__
+ 	// This depends on which version you are compiling for... OSX10.5 doesn't have consts (old OpenSSL!)
+-	unsigned char **certptr = (unsigned char **) &ptr;
+-	//const unsigned char **certptr = (const unsigned char **) &ptr;
++	//unsigned char **certptr = (unsigned char **) &ptr;
++	const unsigned char **certptr = (const unsigned char **) &ptr;
+ #else
+ 	const unsigned char **certptr = (const unsigned char **) &ptr;
+ #endif
+diff --git a/retroshare-gui/src/RetroShare.pro b/retroshare-gui/src/RetroShare.pro
+index adcb5a0..ffbf669 100644
+--- a/retroshare-gui/src/RetroShare.pro
++++ b/retroshare-gui/src/RetroShare.pro
+@@ -140,19 +140,21 @@ win32 {
+ 
+ macx {
+     # ENABLE THIS OPTION FOR Univeral Binary BUILD.
+-    	CONFIG += ppc x86
+-	QMAKE_MACOSX_DEPLOYMENT_TARGET = 10.4
++	# CONFIG += ppc x86
++	QMAKE_MACOSX_DEPLOYMENT_TARGET = 10.6
+ 
+ 	CONFIG += version_detail_bash_script
+ 	LIBS += ../../libretroshare/src/lib/libretroshare.a
+-        LIBS += -lssl -lcrypto -lz -lgpgme -lgpg-error -lassuan
+-	LIBS += ../../../miniupnpc-1.0/libminiupnpc.a
++	LIBS += ../../../libgpg-error-1.10/build/lib/libgpg-error.a
++	LIBS += ../../../gpgme-1.3.1/build/lib/libgpgme.a
++	LIBS += ../../../miniupnpc-1.3/libminiupnpc.a
++	LIBS += -lssl -lcrypto -lz -lassuan
+ 	LIBS += -framework CoreFoundation
+ 	LIBS += -framework Security
+ 
+  #	LIBS += -framework CoreServices
+ 
+-    	INCLUDEPATH += .
++	INCLUDEPATH += . ../../../gpgme-1.3.1/src
+ 	#DEFINES* = MAC_IDLE # for idle feature
+ 	CONFIG -= uitools
+ 

build_scripts/Windows-msys2/build/build.bat

@@ -18,8 +18,6 @@ if errorlevel 1 goto error_env
 if not "%ParamNoupdate%"=="1" (
 	:: Install needed things
 	%EnvMSYS2Cmd% "pacman --noconfirm --needed -S make git mingw-w64-%RsMSYS2Architecture%-toolchain mingw-w64-%RsMSYS2Architecture%-qt5 mingw-w64-%RsMSYS2Architecture%-miniupnpc mingw-w64-%RsMSYS2Architecture%-sqlcipher mingw-w64-%RsMSYS2Architecture%-cmake mingw-w64-%RsMSYS2Architecture%-rapidjson"
-	:: rnp
-	%EnvMSYS2Cmd% "pacman --noconfirm --needed -S mingw-w64-%RsMSYS2Architecture%-json-c mingw-w64-%RsMSYS2Architecture%-libbotan"
 
 	:: Webui
 	if "%ParamWebui%"=="1" %EnvMSYS2Cmd% "pacman --noconfirm --needed -S mingw-w64-%RsMSYS2Architecture%-doxygen"
@@ -34,16 +32,6 @@ if not "%ParamNoupdate%"=="1" (
 	if "%ParamIndexing%"=="1" %EnvMSYS2Cmd% "pacman --noconfirm --needed -S mingw-w64-%RsMSYS2Architecture%-xapian-core mingw-w64-%RsMSYS2Architecture%-libvorbis mingw-w64-%RsMSYS2Architecture%-flac mingw-w64-%RsMSYS2Architecture%-taglib"
 )
 
-:: Fix webui compilation (TODO: remove when whole RS switched to cmake)
-if "%ParamWebui%"=="1" (
-	pushd "%SourcePath%"
-	copy "%SourcePath%\libretroshare\src\jsonapi\jsonapi-generator-doxygen.conf" "%SourcePath%\jsonapi-generator\src\jsonapi-generator-doxygen.conf" %Quite%
-	copy "%SourcePath%\libretroshare\src\jsonapi\async-method-wrapper-template.cpp.tmpl" "%SourcePath%\jsonapi-generator\src\async-method-wrapper-template.cpp.tmpl" %Quite%
-	copy "%SourcePath%\libretroshare\src\jsonapi\method-wrapper-template.cpp.tmpl" "%SourcePath%\jsonapi-generator\src\method-wrapper-template.cpp.tmpl" %Quite%
-	git update-index --assume-unchanged "jsonapi-generator\src\jsonapi-generator-doxygen.conf" "jsonapi-generator\src\async-method-wrapper-template.cpp.tmpl" "jsonapi-generator\src\method-wrapper-template.cpp.tmpl"
-	popd
-)
-
 :: Initialize environment
 call "%~dp0env.bat" %*
 if errorlevel 2 exit /B 2
@@ -78,7 +66,6 @@ echo %RS_QMAKE_CONFIG% > buildinfo.txt
 echo %RsBuildConfig% >> buildinfo.txt
 echo %RsArchitecture% >> buildinfo.txt
 echo Qt %QtVersion% >> buildinfo.txt
-echo %RsToolchain% >> buildinfo.txt
 echo %RsCompiler% >> buildinfo.txt
 
 call "%ToolsPath%\msys2-path.bat" "%SourcePath%" MSYS2SourcePath
@@ -99,6 +86,11 @@ title Build - %SourceName%-%RsBuildConfig% [make]
 %EnvMSYS2Cmd% "make -j %CoreCount%"
 if errorlevel 1 goto error
 
+:: Webui
+if "%ParamWebui%"=="1" (
+	call "%~dp0..\tools\webui.bat"
+)
+
 :error
 popd
 

build_scripts/Windows-msys2/build/env-base.bat

@@ -1,4 +1,6 @@
 :: Process commandline parameter
+set Param32=0
+set Param64=0
 set ParamRelease=0
 set ParamDebug=0
 set ParamAutologin=0
@@ -7,40 +9,17 @@ set ParamTor=0
 set ParamWebui=0
 set ParamClang=0
 set ParamIndexing=0
-set ParamFriendserver=0
 set ParamNoupdate=0
 set CoreCount=%NUMBER_OF_PROCESSORS%
 set RS_QMAKE_CONFIG=
-set RsToolchain=
-set tcc=0
 
 :parameter_loop
 if "%~1" NEQ "" (
 	for /f "tokens=1,2 delims==" %%a in ("%~1") do (
 		if "%%~a"=="32" (
-			set RsToolchain=mingw32
-			set /A tcc=tcc+1
+			set Param32=1
 		) else if "%%~a"=="64" (
-			set RsToolchain=mingw64
-			set /A tcc=tcc+1
-		) else if "%%~a"=="mingw32" (
-			set RsToolchain=mingw32
-			set /A tcc=tcc+1
-		) else if "%%~a"=="mingw64" (
-			set RsToolchain=mingw64
-			set /A tcc=tcc+1
-		) else if "%%~a"=="ucrt64" (
-			set RsToolchain=ucrt64
-			set /A tcc=tcc+1
-		) else if "%%~a"=="clang64" (
-			set RsToolchain=clang64
-			set /A tcc=tcc+1
-		) else if "%%~a"=="clang32" (
-			set RsToolchain=clang32
-			set /A tcc=tcc+1
-		) else if "%%~a"=="clangarm64" (
-			set RsToolchain=clangarm64
-			set /A tcc=tcc+1
+			set Param64=1
 		) else if "%%~a"=="release" (
 			set ParamRelease=1
 		) else if "%%~a"=="debug" (
@@ -59,8 +38,6 @@ if "%~1" NEQ "" (
 			set ParamClang=1
 		) else if "%%~a"=="indexing" (
 			set ParamIndexing=1
-		) else if "%%~a"=="friendserver" (
-			set ParamFriendserver=1
 		) else if "%%~a"=="noupdate" (
 			set ParamNoupdate=1
 		) else if "%%~a"=="CONFIG+" (
@@ -75,37 +52,22 @@ if "%~1" NEQ "" (
 	goto parameter_loop
 )
 
-if %tcc% NEQ 1 (
-	echo Multiple or no toolchain specified
-	goto :usage
-)
+if "%Param32%"=="1" (
+	if "%Param64%"=="1" (
+		echo.
+		echo 32-bit or 64-bit?
+		goto :usage
+	)
 
-if "%RsToolchain%"=="mingw32" (
+	set RsBit=32
 	set RsArchitecture=x86
 	set RsMSYS2Architecture=i686
-	set MSYSTEM=MINGW32
-) else if "%RsToolchain%"=="mingw64" (
+)
+
+if "%Param64%"=="1" (
+	set RsBit=64
 	set RsArchitecture=x64
 	set RsMSYS2Architecture=x86_64
-	set MSYSTEM=MINGW64
-) else if "%RsToolchain%"=="ucrt64" (
-	set RsArchitecture=x64
-	set RsMSYS2Architecture=ucrt-x86_64
-	set MSYSTEM=UCRT64
-) else if "%RsToolchain%"=="clang64" (
-	set RsArchitecture=x64
-	set RsMSYS2Architecture=clang-x86_64
-	set MSYSTEM=CLANG64
-	set ParamClang=1
-) else if "%RsToolchain%"=="clang32" (
-	set RsArchitecture=x86
-	set RsMSYS2Architecture=clang-i686
-	set MSYSTEM=CLANG32
-	set ParamClang=1
-) else if "%RsToolchain%"=="clangarm64" (
-	set RsArchitecture=arm64
-	set RsMSYS2Architecture=clang-aarch64
-	set MSYSTEM=CLANGARM64
 )
 
 if "%ParamClang%"=="1" (
@@ -114,6 +76,8 @@ if "%ParamClang%"=="1" (
 	set RsCompiler=GCC
 )
 
+if "%RsBit%"=="" goto :usage
+
 if "%ParamRelease%"=="1" (
 	if "%ParamDebug%"=="1" (
 		echo.
@@ -140,21 +104,14 @@ if "%ParamIndexing%"=="1" (
 	set RS_QMAKE_CONFIG=%RS_QMAKE_CONFIG% "CONFIG+=rs_deep_channels_index" "CONFIG+=rs_deep_files_index" "CONFIG+=rs_deep_files_index_ogg" "CONFIG+=rs_deep_files_index_flac" "CONFIG+=rs_deep_files_index_taglib"
 )
 
-if "%ParamFriendserver%"=="1" (
-	set RS_QMAKE_CONFIG=%RS_QMAKE_CONFIG% "CONFIG+=rs_efs"
-)
-
 exit /B 0
 
 :usage
 echo.
-echo Usage: 32^|64^|other release^|debug [autologin plugins webui singlethread clang indexing friendserver noupdate] ["CONFIG+=..."]
+echo Usage: 32^|64 release^|debug [autologin plugins webui singlethread clang indexing noupdate] ["CONFIG+=..."]
 echo.
 echo Mandatory parameter
-echo 32^|64              32-bit or 64-bit version (same as mingw32 or mingw64)
-echo                    Or you can specify any other toolchain supported by msys2:
-echo                         mingw32^|mingw64^|clang32^|clang64^|ucrt64^|clangarm64
-echo                         More info: https://www.msys2.org/docs/environments
+echo 32^|64              32-bit or 64-bit Version
 echo release^|debug      Build release or debug version
 echo.
 echo Optional parameter (need clean when changed)
@@ -164,7 +121,6 @@ echo webui              Enable JsonAPI and pack webui files
 echo singlethread       Use only 1 thread for building
 echo clang              Use clang compiler instead of GCC
 echo indexing           Build with deep channel and file indexing support
-echo friendserver       Enable friendserver support
 echo noupdate           Skip updating the libraries
 echo "CONFIG+=..."      Enable some extra features, you can find the almost complete list in retroshare.pri
 echo.

build_scripts/Windows-msys2/build/env.bat

@@ -2,6 +2,8 @@ call "%~dp0env-base.bat" %*
 if errorlevel 2 exit /B 2
 if errorlevel 1 goto error_env
 
+set MSYSTEM=MINGW%RsBit%
+
 set BuildPath=%EnvRootPath%\builds
 set DeployPath=%EnvRootPath%\deploy
 
@@ -12,13 +14,13 @@ if not exist "%DeployPath%" mkdir "%DeployPath%"
 call "%ToolsPath%\get-qt-version.bat" QtVersion
 if "%QtVersion%"=="" %cecho% error "Cannot get Qt version." & exit /B 1
 
-set RsMinGWPath=%EnvMSYS2BasePath%\%RsToolchain%
+set RsMinGWPath=%EnvMSYS2BasePath%\mingw%RsBit%
 
-set RsBuildPath=%BuildPath%\Qt-%QtVersion%-%RsToolchain%-%RsCompiler%-%RsBuildConfig%
-set RsDeployPath=%DeployPath%\Qt-%QtVersion%%RsType%-%RsToolchain%-%RsCompiler%-%RsBuildConfig%
+set RsBuildPath=%BuildPath%\Qt-%QtVersion%-%RsArchitecture%-%RsCompiler%-%RsBuildConfig%
+set RsDeployPath=%DeployPath%\Qt-%QtVersion%%RsType%-%RsArchitecture%-%RsCompiler%-%RsBuildConfig%
 set RsPackPath=%DeployPath%
 set RsArchiveAdd=
-set RsWebuiBuildPath=%RsBuildPath%\retroshare-webui\webui
+set RsWebuiPath=%RootPath%\%SourceName%-webui
 
 if not exist "%~dp0env-mod.bat" goto no_mod
 call "%~dp0env-mod.bat"

build_scripts/Windows-msys2/build/pack.bat

@@ -104,11 +104,6 @@ copy "%RsBuildPath%\retroshare-gui\src\%RsBuildConfig%\RetroShare*.exe" "%RsDepl
 copy "%RsBuildPath%\retroshare-nogui\src\%RsBuildConfig%\retroshare*-nogui.exe" "%RsDeployPath%" %Quite%
 copy "%RsBuildPath%\retroshare-service\src\%RsBuildConfig%\retroshare*-service.exe" "%RsDeployPath%" %Quite%
 copy "%RsBuildPath%\supportlibs\cmark\build\src\libcmark.dll" "%RsDeployPath%" %Quite%
-if exist "%RsBuildPath%\libretroshare\src\lib\retroshare.dll" copy "%RsBuildPath%\libretroshare\src\lib\retroshare.dll" "%RsDeployPath%" %Quite%
-if exist "%RsBuildPath%\supportlibs\librnp\Build\src\lib\librnp.dll" copy "%RsBuildPath%\supportlibs\librnp\Build\src\lib\librnp.dll" "%RsDeployPath%" %Quite%
-if exist "%RsBuildPath%\retroshare-friendserver\src\%RsBuildConfig%\retroshare-friendserver.exe" (
-	copy "%RsBuildPath%\retroshare-friendserver\src\%RsBuildConfig%\retroshare-friendserver.exe" "%RsDeployPath%" %Quite%
-)
 
 echo copy extensions
 for /D %%D in ("%RsBuildPath%\plugins\*") do (
@@ -136,14 +131,8 @@ del /Q "%RsDeployPath%\imageformats\*d?.dll" %Quite%
 
 if "%ParamTor%"=="1" (
 	echo copy tor
-	if not exist "%RsDeployPath%\tor" mkdir "%RsDeployPath%\tor"
-	copy "%RsMinGWPath%\bin\tor.exe" "%RsDeployPath%\tor" %Quite%
-	copy "%RsMinGWPath%\bin\tor-gencert.exe" "%RsDeployPath%\tor" %Quite%
-
-	echo copy tor dependencies
-	for /R "%RsDeployPath%\tor" %%D in (*.exe) do (
-		call :copy_dependencies "%%D" "%RsDeployPath%\tor"
-	)
+	copy "%RsMinGWPath%\bin\tor.exe" "%RsDeployPath%" %Quite%
+	copy "%RsMinGWPath%\bin\tor-gencert.exe" "%RsDeployPath%" %Quite%
 )
 
 echo copy dependencies
@@ -151,15 +140,14 @@ for /R "%RsDeployPath%" %%D in (*.dll, *.exe) do (
 	call :copy_dependencies "%%D" "%RsDeployPath%"
 )
 
-if exist "%SourcePath%\retroshare-gui\src\qss" (
-	echo copy qss
-	xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
-)
+echo copy qss
+xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
 
 echo copy stylesheets
 xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite%
 rmdir /S /Q "%RsDeployPath%\stylesheets\compact" %Quite%
 rmdir /S /Q "%RsDeployPath%\stylesheets\standard" %Quite%
+rmdir /S /Q "%RsDeployPath%\stylesheets\__MACOSX__Bubble" %Quite%
 
 echo copy sounds
 xcopy /S "%SourcePath%\retroshare-gui\src\sounds" "%RsDeployPath%\sounds" %Quite%
@@ -188,12 +176,12 @@ echo copy buildinfo.txt
 copy "%RsBuildPath%\buildinfo.txt" "%RsDeployPath%" %Quite%
 
 if "%ParamWebui%"=="1" (
-	if exist "%RsWebuiBuildPath%" (
+	if exist "%RsWebuiPath%\webui" (
 		echo copy webui
 		mkdir "%RsDeployPath%\webui"
-		xcopy /S "%RsWebuiBuildPath%" "%RsDeployPath%\webui" %Quite%
+		xcopy /S "%RsWebuiPath%\webui" "%RsDeployPath%\webui" %Quite%
 	) else (
-		%cecho% error "Webui is enabled, but no webui data found at %RsWebuiBuildPath%"
+		%cecho% error "Webui is enabled, but no webui data found at %RsWebuiPath%\webui"
 		goto error
 	)
 )
@@ -224,8 +212,8 @@ endlocal
 exit /B 1
 
 :copy_extension
-if exist "%~1\lib\%~n1.dll" (
-	copy "%~1\lib\%~n1.dll" %2 %Quite%
+if exist "%~1\%RsBuildConfig%\%~n1.dll" (
+	copy "%~1\%RsBuildConfig%\%~n1.dll" %2 %Quite%
 )
 goto :EOF
 

build_scripts/Windows-msys2/env/tools/prepare-msys2.bat

@@ -16,13 +16,7 @@ if "%~1"=="clean" (
 	goto exit
 )
 
-set MSYS2Version=20241208
-
-set MSYS2Install=msys2-base-x86_64-%MSYS2Version%.sfx.exe
-set MSYS2Url=https://github.com/msys2/msys2-installer/releases/download/%MSYS2Version:~0,4%-%MSYS2Version:~4,2%-%MSYS2Version:~6,2%/%MSYS2Install%
-set MSYS2UnpackPath=%EnvMSYS2Path%\msys64
-
-if exist "%MSYS2UnpackPath%\usr\bin\pacman.exe" (
+if exist "%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\pacman.exe" (
 	if "%~1"=="reinstall" (
 		choice /M "Found existing MSYS2 version. Do you want to proceed?"
 		if !ERRORLEVEL!==2 goto exit
@@ -31,26 +25,28 @@ if exist "%MSYS2UnpackPath%\usr\bin\pacman.exe" (
 	)
 )
 
-if exist "%MSYS2UnpackPath%" (
-	%cecho% info "Remove previous MSYS2 version"
-	call "%ToolsPath%\remove-dir.bat" "%MSYS2UnpackPath%"
-)
+set MSYS2Install=msys2-base-%MSYS2Architecture%-20200720.tar.xz
+set MSYS2Url=http://sourceforge.net/projects/msys2/files/Base/%MSYS2Architecture%/%MSYS2Install%/download
 
-%cecho% info "Download MSYS2 installation files"
+%cecho% info "Remove previous MSYS2 version"
+call "%ToolsPath%\remove-dir.bat" "%EnvMSYS2Path%"
+
+%cecho% info "Download installation files"
 if not exist "%EnvDownloadPath%\%MSYS2Install%" call "%ToolsPath%\download-file.bat" "%MSYS2Url%" "%EnvDownloadPath%\%MSYS2Install%"
 if not exist "%EnvDownloadPath%\%MSYS2Install%" %cecho% error "Cannot download MSYS" & goto error
 
 %cecho% info "Unpack MSYS2"
-"%EnvDownloadPath%\%MSYS2Install%" -y -o"%EnvMSYS2Path%"
+"%EnvSevenZipExe%" x -so "%EnvDownloadPath%\%MSYS2Install%" | "%EnvSevenZipExe%" x -y -si -ttar -o"%EnvMSYS2Path%"
 
-set MSYS2SH=%MSYS2UnpackPath%\usr\bin\sh
+set MSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh
 
 %cecho% info "Initialize MSYS2"
-"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
-"%MSYS2SH%" -lc "pacman --noconfirm -Sy"
-"%MSYS2SH%" -lc "pacman --noconfirm -Su"
+"%MSYS2SH%" -lc "pacman -Sy"
+"%MSYS2SH%" -lc "pacman --noconfirm --needed -S bash pacman pacman-mirrors msys2-runtime"
 
-call "%MSYS2UnpackPath%\autorebase.bat"
+call "%EnvMSYS2Path%\msys%MSYS2Base%\autorebase.bat"
+call "%EnvRootPath%\update-msys2.bat"
+call "%EnvRootPath%\update-msys2.bat"
 
 :exit
 endlocal

build_scripts/Windows-msys2/env/tools/prepare-tools.bat

@@ -34,7 +34,7 @@ if not exist "%EnvToolsPath%\cecho.exe" (
 	if not exist "%EnvDownloadPath%\%cCEhoInstall%" echo Cannot download cecho installation& goto error
 
 	echo Unpack cecho
-	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%CEchoInstall%" -y -bso0
+	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%CEchoInstall%"
 	copy "%EnvTempPath%\cecho.exe" "%EnvToolsPath%"
 
 	call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%"

build_scripts/Windows-msys2/env/tools/root/update-msys2.bat

@@ -2,13 +2,18 @@
 
 setlocal
 
-if not exist "%~dp0msys2\msys64" goto :EOF
+if exist "%~dp0msys2\msys32" call :update 32
+if exist "%~dp0msys2\msys64" call :update 64
 
-set MSYS2SH=%~dp0msys2\msys64\usr\bin\sh
+goto :EOF
 
-echo Update MSYS2
-"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
-"%MSYS2SH%" -lc "pacman --noconfirm -Su"
+:update
+set MSYSSH=%~dp0msys2\msys%~1\usr\bin\sh
 
+echo Update MSYS2 %~1
+"%MSYSSH%" -lc "pacman -Sy"
+"%MSYSSH%" -lc "pacman --noconfirm -Su"
+
+:exit
 endlocal
 goto :EOF

build_scripts/Windows-msys2/installer/retroshare-Qt5.nsi

@@ -218,7 +218,7 @@ Section $(Section_Data) Section_Data
 
   ; Stylesheets
   SetOutPath "$INSTDIR\qss"
-  File /nonfatal /r "${DEPLOYDIR}\qss\*.*"
+  File /r "${DEPLOYDIR}\qss\*.*"
 SectionEnd
 
 ;Section $(Section_Link) Section_Link

build_scripts/Windows-msys2/tools/webui.bat

@@ -0,0 +1,22 @@
+setlocal
+
+echo.
+echo === webui
+echo.
+title Build webui
+
+if not exist "%RsWebuiPath%" (
+	echo Checking out webui source into %RsWebuiPath%
+	if not "%ParamNoupdate%"=="1" (
+		%EnvMSYS2Cmd% "pacman --noconfirm --needed -S git"
+	)
+	git clone https://github.com/RetroShare/RSNewWebUI.git "%RsWebuiPath%"
+) else (
+	echo Webui source found at %RsWebuiPath%		
+)
+
+pushd "%RsWebuiPath%\webui-src\make-src"
+%EnvMSYS2Cmd% "sh build.sh"
+popd
+
+endlocal

build_scripts/Windows/build-libs/Makefile

@@ -1,30 +1,25 @@
-ZLIB_VERSION=1.2.11
-BZIP2_VERSION=1.0.8
-MINIUPNPC_VERSION=2.2.3
-OPENSSL_VERSION=1.1.1w
+ZLIB_VERSION=1.2.3
+BZIP2_VERSION=1.0.6
+MINIUPNPC_VERSION=2.0
+OPENSSL_VERSION=1.1.1g
 SPEEX_VERSION=1.2.0
-SPEEXDSP_VERSION=1.2.0
-LIBXML2_VERSION=2.9.12
-LIBXSLT_VERSION=1.1.34
-CURL_VERSION=7.81.0
+SPEEXDSP_VERSION=1.2rc3
+OPENCV_VERSION=3.4.11
+LIBXML2_VERSION=2.9.7
+LIBXSLT_VERSION=1.1.32
+CURL_VERSION=7.58.0
 TCL_VERSION=8.6.10
-SQLCIPHER_VERSION=4.5.0
-LIBMICROHTTPD_VERSION=0.9.75
-FFMPEG_VERSION=4.4
+SQLCIPHER_VERSION=4.4.0
+LIBMICROHTTPD_VERSION=0.9.59
+FFMPEG_VERSION=3.4
 RAPIDJSON_VERSION=1.1.0
-XAPIAN_VERSION=1.4.19
-#RNP_VERSION=0.17.1
-
-# libaries for rnp
-JSON_C_VERSION=0.18
-BOTAN_VERSION=2.19.5
+XAPIAN_VERSION=1.4.7
 
 DOWNLOAD_PATH?=download
 BUILD_PATH=build
 LIBS_PATH?=libs
 
-all: dirs zlib bzip2 miniupnpc openssl speex speexdsp libxml2 libxslt curl sqlcipher libmicrohttpd ffmpeg rapidjson xapian jsonc botan copylibs
-#rnp
+all: dirs zlib bzip2 miniupnpc openssl speex speexdsp opencv libxml2 libxslt curl sqlcipher libmicrohttpd ffmpeg rapidjson xapian copylibs
 
 download: \
 	$(DOWNLOAD_PATH)/zlib-$(ZLIB_VERSION).tar.gz \
@@ -33,6 +28,7 @@ download: \
 	$(DOWNLOAD_PATH)/openssl-$(OPENSSL_VERSION).tar.gz \
 	$(DOWNLOAD_PATH)/speex-$(SPEEX_VERSION).tar.gz \
 	$(DOWNLOAD_PATH)/speexdsp-$(SPEEXDSP_VERSION).tar.gz \
+	$(DOWNLOAD_PATH)/opencv-$(OPENCV_VERSION).tar.gz \
 	$(DOWNLOAD_PATH)/libxml2-$(LIBXML2_VERSION).tar.gz \
 	$(DOWNLOAD_PATH)/libxslt-$(LIBXSLT_VERSION).tar.gz \
 	$(DOWNLOAD_PATH)/curl-$(CURL_VERSION).tar.gz \
@@ -62,7 +58,8 @@ $(BUILD_PATH)/zlib-$(ZLIB_VERSION): $(DOWNLOAD_PATH)/zlib-$(ZLIB_VERSION).tar.gz
 	rm -r -f $(BUILD_PATH)/zlib-*
 	tar xvf $(DOWNLOAD_PATH)/zlib-$(ZLIB_VERSION).tar.gz
 	# build
-	cd zlib-$(ZLIB_VERSION) && make -f win32/Makefile.gcc libz.a
+	cd zlib-$(ZLIB_VERSION) && ./configure
+	#cd zlib-$(ZLIB_VERSION) && make install prefix="`pwd`/../$(BUILD_PATH)"
 	cd zlib-$(ZLIB_VERSION) && make
 	# copy files
 	mkdir -p $(BUILD_PATH)/zlib-$(ZLIB_VERSION).tmp/include
@@ -77,7 +74,7 @@ $(BUILD_PATH)/zlib-$(ZLIB_VERSION): $(DOWNLOAD_PATH)/zlib-$(ZLIB_VERSION).tar.gz
 bzip2: $(BUILD_PATH)/bzip2-$(BZIP2_VERSION)
 
 $(DOWNLOAD_PATH)/bzip2-$(BZIP2_VERSION).tar.gz:
-	wget https://www.sourceware.org/pub/bzip2/bzip2-$(BZIP2_VERSION).tar.gz -O $(DOWNLOAD_PATH)/bzip2-$(BZIP2_VERSION).tar.gz
+	wget https://sourceforge.net/projects/bzip2/files/bzip2-$(BZIP2_VERSION).tar.gz/download -O $(DOWNLOAD_PATH)/bzip2-$(BZIP2_VERSION).tar.gz
 
 $(BUILD_PATH)/bzip2-$(BZIP2_VERSION): $(DOWNLOAD_PATH)/bzip2-$(BZIP2_VERSION).tar.gz
 	# prepare
@@ -105,10 +102,10 @@ $(BUILD_PATH)/miniupnpc-$(MINIUPNPC_VERSION): $(DOWNLOAD_PATH)/miniupnpc-$(MINIU
 	rm -r -f $(BUILD_PATH)/miniupnpc-*
 	tar xvf $(DOWNLOAD_PATH)/miniupnpc-$(MINIUPNPC_VERSION).tar.gz
 	# build
-	cd miniupnpc-$(MINIUPNPC_VERSION) && export CC=gcc && export PATH=.:$$PATH && make -f Makefile.mingw libminiupnpc.a miniupnpc.dll
+	cd miniupnpc-$(MINIUPNPC_VERSION) && export CC=gcc && export PATH=.:$$PATH && make -f Makefile.mingw init libminiupnpc.a miniupnpc.dll
 	# copy files
 	mkdir -p $(BUILD_PATH)/miniupnpc-$(MINIUPNPC_VERSION).tmp/include/miniupnpc
-	cp miniupnpc-$(MINIUPNPC_VERSION)/include/*.h $(BUILD_PATH)/miniupnpc-$(MINIUPNPC_VERSION).tmp/include/miniupnpc/
+	cp miniupnpc-$(MINIUPNPC_VERSION)/*.h $(BUILD_PATH)/miniupnpc-$(MINIUPNPC_VERSION).tmp/include/miniupnpc/
 	mkdir -p $(BUILD_PATH)/miniupnpc-$(MINIUPNPC_VERSION).tmp/lib
 	cp miniupnpc-$(MINIUPNPC_VERSION)/miniupnpc.lib $(BUILD_PATH)/miniupnpc-$(MINIUPNPC_VERSION).tmp/lib/
 	mkdir -p $(BUILD_PATH)/miniupnpc-$(MINIUPNPC_VERSION).tmp/bin
@@ -190,6 +187,29 @@ $(BUILD_PATH)/speexdsp-$(SPEEXDSP_VERSION): $(DOWNLOAD_PATH)/speexdsp-$(SPEEXDSP
 	rm -r -f speexdsp-$(SPEEXDSP_VERSION)
 	mv $(BUILD_PATH)/speexdsp-$(SPEEXDSP_VERSION).tmp $(BUILD_PATH)/speexdsp-$(SPEEXDSP_VERSION)
 
+opencv: $(BUILD_PATH)/opencv-$(OPENCV_VERSION)
+
+$(DOWNLOAD_PATH)/opencv-$(OPENCV_VERSION).tar.gz:
+	wget --no-check-certificate https://github.com/opencv/opencv/archive/$(OPENCV_VERSION).tar.gz -O $(DOWNLOAD_PATH)/opencv-$(OPENCV_VERSION).tar.gz
+
+$(BUILD_PATH)/opencv-$(OPENCV_VERSION): $(DOWNLOAD_PATH)/opencv-$(OPENCV_VERSION).tar.gz
+	# prepare
+	rm -r -f $(BUILD_PATH)/opencv-*
+	tar xvf $(DOWNLOAD_PATH)/opencv-$(OPENCV_VERSION).tar.gz
+	# build
+	mkdir -p opencv-$(OPENCV_VERSION)/build
+	#cd opencv-$(OPENCV_VERSION)/build && cmake .. -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Release -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_INSTALL_PREFIX="`pwd`/../../build"
+	cd opencv-$(OPENCV_VERSION)/build && cmake .. -G"MSYS Makefiles" -DCMAKE_BUILD_TYPE=Release -DBUILD_PERF_TESTS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -DENABLE_CXX11=ON -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS} -DSTRSAFE_NO_DEPRECATE" -DCMAKE_INSTALL_PREFIX="`pwd`/install"
+	cd opencv-$(OPENCV_VERSION)/build && make install
+	# copy files
+	mkdir -p $(BUILD_PATH)/opencv-$(OPENCV_VERSION).tmp/include
+	cp -r opencv-$(OPENCV_VERSION)/build/install/include/* $(BUILD_PATH)/opencv-$(OPENCV_VERSION).tmp/include/
+	mkdir -p $(BUILD_PATH)/opencv-$(OPENCV_VERSION).tmp/lib/opencv
+	cp -r opencv-$(OPENCV_VERSION)/build/install/x64/mingw/staticlib/* $(BUILD_PATH)/opencv-$(OPENCV_VERSION).tmp/lib/opencv/
+	# cleanup
+	rm -r -f opencv-$(OPENCV_VERSION)
+	mv $(BUILD_PATH)/opencv-$(OPENCV_VERSION).tmp $(BUILD_PATH)/opencv-$(OPENCV_VERSION)
+
 libxml2: $(BUILD_PATH)/libxml2-$(LIBXML2_VERSION)
 
 $(DOWNLOAD_PATH)/libxml2-$(LIBXML2_VERSION).tar.gz:
@@ -319,7 +339,7 @@ $(BUILD_PATH)/ffmpeg-$(FFMPEG_VERSION): $(DOWNLOAD_PATH)/ffmpeg-$(FFMPEG_VERSION
 	rm -r -f $(BUILD_PATH)/ffmpeg-*
 	tar xvf $(DOWNLOAD_PATH)/ffmpeg-$(FFMPEG_VERSION).tar.gz
 	# build
-	cd ffmpeg-$(FFMPEG_VERSION) && ./configure --disable-shared --enable-static --disable-programs --disable-ffmpeg --disable-ffplay --disable-ffprobe --disable-doc --disable-htmlpages --disable-manpages --disable-podpages --disable-txtpages --disable-yasm --disable-everything --enable-encoder=mpeg4 --enable-decoder=mpeg4 --prefix="`pwd`/../$(BUILD_PATH)/ffmpeg-$(FFMPEG_VERSION).tmp"
+	cd ffmpeg-$(FFMPEG_VERSION) && ./configure --disable-shared --enable-static --disable-programs --disable-ffmpeg --disable-ffplay --disable-ffprobe --disable-ffserver --disable-doc --disable-htmlpages --disable-manpages --disable-podpages --disable-txtpages --disable-yasm --disable-everything --enable-encoder=mpeg4 --enable-decoder=mpeg4 --prefix="`pwd`/../$(BUILD_PATH)/ffmpeg-$(FFMPEG_VERSION).tmp"
 	cd ffmpeg-$(FFMPEG_VERSION) && make install
 	# cleanup
 	rm -r -f ffmpeg-$(FFMPEG_VERSION)
@@ -369,57 +389,6 @@ $(BUILD_PATH)/xapian-core-$(XAPIAN_VERSION): $(DOWNLOAD_PATH)/xapian-core-$(XAPI
 	rm -r -f xapian-core-$(XAPIAN_VERSION)
 	mv $(BUILD_PATH)/xapian-core-$(XAPIAN_VERSION).tmp $(BUILD_PATH)/xapian-core-$(XAPIAN_VERSION)
 
-jsonc: $(BUILD_PATH)/json-c-$(JSON_C_VERSION)
-
-$(BUILD_PATH)/json-c-$(JSON_C_VERSION):
-	# prepare
-	rm -r -f $(BUILD_PATH)/json-c-*
-	[ -d "json-c-$(JSON_C_VERSION)" ] || git clone https://github.com/json-c/json-c.git --depth=1 --branch json-c-$(JSON_C_VERSION) "json-c-$(JSON_C_VERSION)"
-	# build
-	mkdir -p json-c-$(JSON_C_VERSION)/build
-	cd json-c-$(JSON_C_VERSION)/build && cmake .. -G"MSYS Makefiles" -Wno-dev -DCMAKE_BUILD_TYPE="release" -DBUILD_SHARED_LIBS=off -DBUILD_STATIC_LIBS=on -DBUILD_TESTING=off -DCMAKE_CXX_FLAGS="-D__MINGW_USE_VC2005_COMPAT" -DCMAKE_INSTALL_PREFIX="`pwd`/install" -DCMAKE_INSTALL_PREFIX="`pwd`/../../$(BUILD_PATH)/json-c-$(JSON_C_VERSION).tmp"
-	cd json-c-$(JSON_C_VERSION)/build && make install
-	# cleanup
-	rm -r -f json-c-$(JSON_C_VERSION)
-	mv $(BUILD_PATH)/json-c-$(JSON_C_VERSION).tmp $(BUILD_PATH)/json-c-$(JSON_C_VERSION)
-
-botan: $(BUILD_PATH)/botan-$(BOTAN_VERSION)
-
-$(BUILD_PATH)/botan-$(BOTAN_VERSION):
-	# prepare
-	pacman --needed --noconfirm -S python3
-	rm -r -f $(BUILD_PATH)/botan-*
-	[ -d "botan-$(BOTAN_VERSION)" ] || git clone https://github.com/randombit/botan.git --depth=1 --branch $(BOTAN_VERSION) "botan-$(BOTAN_VERSION)"
-	# build
-	if [ $(MSYSTEM) = "MINGW32" ] ; then cd botan-$(BOTAN_VERSION) && ./configure.py --os=mingw --cpu=x86_32 --disable-shared-library --enable-static-library --extra-cxxflags="-D__MINGW_USE_VC2005_COMPAT" --prefix="`pwd`/../$(BUILD_PATH)/botan-$(BOTAN_VERSION).tmp" ; fi
-	if [ $(MSYSTEM) = "MINGW64" ] ; then cd botan-$(BOTAN_VERSION) && ./configure.py --os=mingw --cpu=x86_64 --disable-shared-library --enable-static-library --prefix="`pwd`/../$(BUILD_PATH)/botan-$(BOTAN_VERSION).tmp" ; fi
-	cd botan-$(BOTAN_VERSION) && make install
-	# cleanup
-	rm -r -f botan-$(BOTAN_VERSION)
-	mv $(BUILD_PATH)/botan-$(BOTAN_VERSION).tmp $(BUILD_PATH)/botan-$(BOTAN_VERSION)
-
-rnp: $(BUILD_PATH)/rnp-$(RNP_VERSION)
-
-$(BUILD_PATH)/rnp-$(RNP_VERSION):
-	# prepare
-	rm -r -f $(BUILD_PATH)/rnp-*
-	[ -d "rnp-$(RNP_VERSION)" ] || git clone https://github.com/rnpgp/rnp.git --depth=1 --branch v$(RNP_VERSION) --recurse-submodules --shallow-submodules "rnp-$(RNP_VERSION)"
-	# build
-	mkdir -p rnp-$(RNP_VERSION)/build
-	cd rnp-$(RNP_VERSION)/build && cmake .. -G"MSYS Makefiles" -Wno-dev -DCMAKE_INSTALL_PREFIX="`pwd`/install" -DBUILD_SHARED_LIBS=yes -DBUILD_TESTING=off -DCMAKE_CXX_FLAGS="-D__MINGW_USE_VC2005_COMPAT -D__STDC_FORMAT_MACROS" -DBZIP2_INCLUDE_DIR="`pwd`/../../$(BUILD_PATH)/bzip2-$(BZIP2_VERSION)/include" -DBZIP2_LIBRARY_RELEASE="`pwd`/../../$(BUILD_PATH)/bzip2-$(BZIP2_VERSION)/lib/libbz2.a" -DBZIP2_LIBRARIES="`pwd`/../../$(BUILD_PATH)/bzip2-$(BZIP2_VERSION)/lib/libbz2.a" -DZLIB_INCLUDE_DIR="`pwd`/../../$(BUILD_PATH)/zlib-$(ZLIB_VERSION)/include" -DZLIB_LIBRARY="`pwd`/../../$(BUILD_PATH)/zlib-$(ZLIB_VERSION)/lib/libz.a" -DJSON-C_INCLUDE_DIR="`pwd`/../../$(BUILD_PATH)/json-c-$(JSON_C_VERSION)/include/json-c" -DJSON-C_LIBRARY="`pwd`/../../$(BUILD_PATH)/json-c-$(JSON_C_VERSION)/lib/libjson-c.a" -DBOTAN_INCLUDE_DIR="`pwd`/../../$(BUILD_PATH)/botan-$(BOTAN_VERSION)/include/botan-`echo $(BOTAN_VERSION) | cut -c1-1`" -DBOTAN_LIBRARY="`pwd`/../../$(BUILD_PATH)/botan-$(BOTAN_VERSION)/lib/libbotan-`echo $(BOTAN_VERSION) | cut -c1-1`.a"
-	cmake --build rnp-$(RNP_VERSION)/build
-	# copy files
-	mkdir -p $(BUILD_PATH)/rnp-$(RNP_VERSION).tmp/include/rnp
-	cp -r rnp-$(RNP_VERSION)/include/rnp/* $(BUILD_PATH)/rnp-$(RNP_VERSION).tmp/include/rnp/
-	cp -r rnp-$(RNP_VERSION)/build/src/lib/rnp/* $(BUILD_PATH)/rnp-$(RNP_VERSION).tmp/include/rnp/
-	mkdir -p $(BUILD_PATH)/rnp-$(RNP_VERSION).tmp/lib
-	cp -r rnp-$(RNP_VERSION)/build/src/lib/librnp.dll.a $(BUILD_PATH)/rnp-$(RNP_VERSION).tmp/lib/
-	mkdir -p $(BUILD_PATH)/rnp-$(RNP_VERSION).tmp/bin
-	cp -r rnp-$(RNP_VERSION)/build/src/lib/librnp.dll $(BUILD_PATH)/rnp-$(RNP_VERSION).tmp/bin/
-	# cleanup
-	rm -r -f rnp-$(RNP_VERSION)
-	mv $(BUILD_PATH)/rnp-$(RNP_VERSION).tmp $(BUILD_PATH)/rnp-$(RNP_VERSION)
-
 copylibs:
 	rm -r -f $(LIBS_PATH) ; \
 	mkdir -p $(LIBS_PATH) ; \

build_scripts/Windows/build.bat

@@ -13,11 +13,12 @@ call "%~dp0build-libs\build-libs.bat"
 if errorlevel 1 %cecho% error "Failed to build libraries." & exit /B %ERRORLEVEL%
 
 %cecho% info "Build %SourceName%"
-call "%~dp0build\build.bat" release autologin webui plugins nativedialogs service
+call "%~dp0build\build.bat" release autologin jsonapi
+rem plugins
 if errorlevel 1 %cecho% error "Failed to build %SourceName%." & exit /B %ERRORLEVEL%
 
 %cecho% info "Pack %SourceName%"
-call "%~dp0build\pack.bat" release webui plugins service
+call "%~dp0build\pack.bat" release
 if errorlevel 1 %cecho% error "Failed to pack %SourceName%." & exit /B %ERRORLEVEL%
 
 %cecho% info "Build installer"

build_scripts/Windows/build/build-installer.bat

@@ -9,7 +9,7 @@ call "%EnvPath%\env.bat"
 if errorlevel 1 goto error_env
 
 :: Initialize environment
-call "%~dp0env.bat" installer release
+call "%~dp0env.bat" release
 if errorlevel 2 exit /B 2
 if errorlevel 1 goto error_env
 
@@ -40,7 +40,6 @@ set NSIS_PARAM=%NSIS_PARAM% /DARCHITECTURE="%GCCArchitecture%"
 set NSIS_PARAM=%NSIS_PARAM% /DDATE="%RsDate%"
 
 if exist "%EnvTorPath%\Tor\tor.exe" set NSIS_PARAM=%NSIS_PARAM% /DTORDIR="%EnvTorPath%\Tor"
-if exist "%RsWebuiBuildPath%" set NSIS_PARAM=%NSIS_PARAM% /DWEBUIDIR="%RsWebuiBuildPath%"
 
 :: Get compiled version
 call "%ToolsPath%\get-rs-version.bat" "%RsBuildPath%\retroshare-gui\src\%RsBuildConfig%\retroshare.exe" RsVersion

build_scripts/Windows/build/build.bat

@@ -9,7 +9,7 @@ call "%EnvPath%\env.bat"
 if errorlevel 1 goto error_env
 
 :: Initialize environment
-call "%~dp0env.bat" build %*
+call "%~dp0env.bat" %*
 if errorlevel 2 exit /B 2
 if errorlevel 1 goto error_env
 
@@ -52,12 +52,7 @@ title Build - %SourceName%-%RsBuildConfig% [qmake]
 set RS_QMAKE_CONFIG=%RsBuildConfig%
 if "%ParamAutologin%"=="1" set RS_QMAKE_CONFIG=%RS_QMAKE_CONFIG% rs_autologin
 if "%ParamJsonApi%"=="1" set RS_QMAKE_CONFIG=%RS_QMAKE_CONFIG% rs_jsonapi
-if "%ParamWebui%"=="1" set RS_QMAKE_CONFIG=%RS_QMAKE_CONFIG% rs_webui
 if "%ParamPlugins%"=="1" set RS_QMAKE_CONFIG=%RS_QMAKE_CONFIG% retroshare_plugins
-if "%ParamUseNativeDialogs%"=="1" set RS_QMAKE_CONFIG=%RS_QMAKE_CONFIG% rs_use_native_dialogs
-if "%ParamService%" NEQ "1" set RS_QMAKE_CONFIG=%RS_QMAKE_CONFIG% no_retroshare_service
-if "%ParamFriendServer%" NEQ "1" set RS_QMAKE_CONFIG=%RS_QMAKE_CONFIG% no_retroshare_friendserver
-if "%ParamEmbeddedFriendServer%"=="1" set RS_QMAKE_CONFIG=%RS_QMAKE_CONFIG% rs_efs
 
 qmake "%SourcePath%\RetroShare.pro" -r -spec win32-g++ "CONFIG+=%RS_QMAKE_CONFIG%" "EXTERNAL_LIB_DIR=%BuildLibsPath%\libs"
 if errorlevel 1 goto error

build_scripts/Windows/build/clean.bat

@@ -8,7 +8,7 @@ if errorlevel 1 goto error_env
 call "%EnvPath%\env.bat"
 if errorlevel 1 goto error_env
 
-call "%~dp0env.bat" clean %*
+call "%~dp0env.bat" %*
 if errorlevel 2 exit /B 2
 if errorlevel 1 goto error_env
 

build_scripts/Windows/build/env.bat

@@ -4,18 +4,10 @@ set ParamDebug=0
 set ParamAutologin=0
 set ParamPlugins=0
 set ParamJsonApi=0
-set ParamWebui=0
-set ParamService=0
-set ParamFriendServer=0
-set ParamEmbeddedFriendServer=0
-set ParamUseNativeDialogs=0
 set ParamTor=0
 set NonInteractive=0
 set CoreCount=%NUMBER_OF_PROCESSORS%
 
-set Module=%~1
-shift /1
-
 :parameter_loop
 if "%~1" NEQ "" (
 	for /f "tokens=1,2 delims==" %%a in ("%~1") do (
@@ -27,15 +19,6 @@ if "%~1" NEQ "" (
 			set ParamAutologin=1
 		) else if "%%~a"=="jsonapi" (
 			set ParamJsonApi=1
-		) else if "%%~a"=="webui" (
-			set ParamJsonApi=1
-			set ParamWebui=1
-		) else if "%%~a"=="service" (
-			set ParamService=1
-		) else if "%%~a"=="friendserver" (
-			set ParamFriendServer=1
-		) else if "%%~a"=="embedded-friendserver" (
-			set ParamEmbeddedFriendServer=1
 		) else if "%%~a"=="plugins" (
 			set ParamPlugins=1
 		) else if "%%~a"=="tor" (
@@ -44,11 +27,9 @@ if "%~1" NEQ "" (
 			set NonInteractive=1
 		) else if "%%~a"=="singlethread" (
 			set CoreCount=1
-		) else if "%%~a"=="nativedialogs" (
-			set ParamUseNativeDialogs=1
 		) else (
 			echo.
-			echo Unknown parameter %1 for %Module%
+			echo Unknown parameter %1
 			goto :usage
 		)
 	)
@@ -105,7 +86,6 @@ set RsBuildPath=%BuildPath%\Qt-%QtVersion%-%GCCArchitecture%-%RsBuildConfig%
 set RsDeployPath=%DeployPath%\Qt-%QtVersion%-%GCCArchitecture%%RsType%-%RsBuildConfig%
 set RsPackPath=%DeployPath%
 set RsArchiveAdd=
-set RsWebuiBuildPath=%RsBuildPath%\retroshare-webui\webui
 
 if not exist "%~dp0env-mod.bat" goto no_mod
 call "%~dp0env-mod.bat"
@@ -116,36 +96,23 @@ exit /B 0
 
 :usage
 echo.
-echo Usage: release^|debug [^<optional parameters^>]
+echo Usage: release^|debug [version autologin plugins]
 echo.
 echo Mandatory parameter
-echo release^|debug         Build release or debug version
+echo release^|debug      Build release or debug version
 echo.
-if "%Module%"=="build" (
-	echo Optional parameter ^(need clean when changed^)
-	echo autologin             Build with autologin
-	echo jsonapi               Build with jsonapi
-	echo webui                 Build with jsonapi and webui
-	echo service               Build service
-	echo friendserver          Build Friend Server
-	echo embedded-friendserver Build with embedded Friend Server
-	echo plugins               Build plugins
-	echo nativedialogs         Build with native dialogs
-	echo.
-	echo Optional parameter
-	echo singlethread          Use only 1 thread for building
-)
-if "%Module%"=="pack" (
-	echo Optional parameter
-	echo webui                 Pack webui
-	echo service               Pack service
-	echo friendserver          Pack Friend Server ^(needs Tor^)
-	echo tor                   Pack Tor version
-	echo plugins               Pack plugins
-)
-if "%Module%"=="git-log" (
-	echo Optional parameter
-	echo non-interactive       Non-interactive mode
-)
+echo Optional parameter (need clean when changed)
+echo autologin          Build with autologin
+echo jsonapi            Build with jsonapi
+echo plugins            Build plugins
+echo.
+echo Optional parameter
+echo singlethread       Use only 1 thread for building
+echo.
+echo Parameter for pack
+echo tor                Pack tor version
+echo.
+echo Parameter for git-log
+echo non-interactive    Non-interactive mode
 echo.
 exit /B 2

build_scripts/Windows/build/git-log.bat

@@ -8,7 +8,7 @@ if errorlevel 1 goto error_env
 call "%EnvPath%\env.bat"
 if errorlevel 1 goto error_env
 
-call "%~dp0env.bat" git-log %*
+call "%~dp0env.bat" %*
 if errorlevel 2 exit /B 2
 if errorlevel 1 goto error_env
 
@@ -88,10 +88,6 @@ echo %RsRef%>"%RsLastRefFile%"
 
 exit /B %ERRORLEVEL%
 
-:error
-%cecho% error "\n%~n0 failed\n"
-exit /B 1
-
 :error_env
 echo Failed to initialize environment.
 endlocal

build_scripts/Windows/build/pack.bat

@@ -11,7 +11,7 @@ call "%EnvPath%\env.bat"
 if errorlevel 1 goto error_env
 
 :: Initialize environment
-call "%~dp0env.bat" pack %*
+call "%~dp0env.bat" %*
 if errorlevel 2 exit /B 2
 if errorlevel 1 goto error_env
 
@@ -95,30 +95,12 @@ copy nul "%RsDeployPath%\portable" %Quite%
 
 echo copy binaries
 copy "%RsBuildPath%\retroshare-gui\src\%RsBuildConfig%\retroshare*.exe" "%RsDeployPath%" %Quite%
-if exist "%RsBuildPath%\libretroshare\src\lib\retroshare.dll" copy "%RsBuildPath%\libretroshare\src\lib\retroshare.dll" "%RsDeployPath%" %Quite%
-if exist "%RsBuildPath%\supportlibs\librnp\Build\src\lib\librnp.dll" copy "%RsBuildPath%\supportlibs\librnp\Build\src\lib\librnp.dll" "%RsDeployPath%" %Quite%
-
-if "%ParamService%"=="1" (
-	copy "%RsBuildPath%\retroshare-service\src\%RsBuildConfig%\retroshare*-service.exe" "%RsDeployPath%" %Quite%
-	if errorlevel 1 %cecho% error "Service not found"& goto error
-)
-
-if "%ParamFriendServer%"=="1" (
-	if "%ParamTor%"=="1" (
-		copy "%RsBuildPath%\retroshare-friendserver\src\%RsBuildConfig%\retroshare-friendserver.exe" "%RsDeployPath%" %Quite%
-		if errorlevel 1 %cecho% error "Friend Server not found"& goto error
-	) else (
-		%cecho% error "Friend Server needs Tor"
-		goto error
-	)
-)
+copy "%RsBuildPath%\retroshare-service\src\%RsBuildConfig%\retroshare*-service.exe" "%RsDeployPath%" %Quite%
 
 echo copy extensions
-if "%ParamPlugins%"=="1" (
-	for /D %%D in ("%RsBuildPath%\plugins\*") do (
-		call :copy_extension "%%D" "%RsDeployPath%\Data\%Extensions%"
-		call :copy_dependencies "%RsDeployPath%\Data\%Extensions%\%%~nxD.dll" "%RsDeployPath%"
-	)
+for /D %%D in ("%RsBuildPath%\plugins\*") do (
+	call :copy_extension "%%D" "%RsDeployPath%\Data\%Extensions%"
+	call :copy_dependencies "%RsDeployPath%\Data\%Extensions%\%%~nxD.dll" "%RsDeployPath%"
 )
 
 echo copy external binaries
@@ -126,7 +108,6 @@ copy "%BuildLibsPath%\libs\bin\*.dll" "%RsDeployPath%" %Quite%
 
 echo copy dependencies
 call :copy_dependencies "%RsDeployPath%\retroshare.exe" "%RsDeployPath%"
-if exist "%RsDeployPath%\retroshare.dll" call :copy_dependencies "%RsDeployPath%\retroshare.dll" "%RsDeployPath%"
 
 echo copy Qt DLL's
 copy "%QtPath%\Qt%QtMainVersion1%Svg%QtMainVersion2%.dll" "%RsDeployPath%" %Quite%
@@ -147,21 +128,17 @@ if exist "%QtPath%\..\plugins\styles\qwindowsvistastyle.dll" (
 copy "%QtPath%\..\plugins\imageformats\*.dll" "%RsDeployPath%\imageformats" %Quite%
 del /Q "%RsDeployPath%\imageformats\*d?.dll" %Quite%
 
-if exist "%SourcePath%\retroshare-gui\src\qss" (
-	echo copy qss
-	xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
-)
+echo copy qss
+xcopy /S "%SourcePath%\retroshare-gui\src\qss" "%RsDeployPath%\qss" %Quite%
 
 echo copy stylesheets
 xcopy /S "%SourcePath%\retroshare-gui\src\gui\qss\chat" "%RsDeployPath%\stylesheets" %Quite%
 rmdir /S /Q "%RsDeployPath%\stylesheets\compact" %Quite%
 rmdir /S /Q "%RsDeployPath%\stylesheets\standard" %Quite%
+rmdir /S /Q "%RsDeployPath%\stylesheets\__MACOSX__Bubble" %Quite%
 
 echo copy sounds
 xcopy /S "%SourcePath%\retroshare-gui\src\sounds" "%RsDeployPath%\sounds" %Quite%
-if "%ParamPlugins%"=="1" (
-	xcopy /S "%SourcePath%\plugins\Voip\gui\sounds" "%RsDeployPath%\sounds" %Quite%
-)
 
 echo copy license
 xcopy /S "%SourcePath%\retroshare-gui\src\license" "%RsDeployPath%\license" %Quite%
@@ -182,17 +159,15 @@ copy "%SourcePath%\libbitdht\src\bitdht\bdboot.txt" "%RsDeployPath%" %Quite%
 echo copy changelog.txt
 copy "%RsBuildPath%\changelog.txt" "%RsDeployPath%" %Quite%
 
-if defined ParamWebui (
+if exist "%SourcePath%\libresapi\src\webui" (
 	echo copy webui
 	mkdir "%RsDeployPath%\webui"
-	xcopy /S "%RsWebuiBuildPath%" "%RsDeployPath%\webui" %Quite%
-	if errorlevel 1 %cecho% error "WebUi not found"& goto error
+	xcopy /S "%SourcePath%\libresapi\src\webui" "%RsDeployPath%\webui" %Quite%
 )
 
 if "%ParamTor%"=="1" (
 	echo copy tor
-	if not exist "%RsDeployPath%\tor" mkdir "%RsDeployPath%\tor"
-	echo n | copy /-y "%EnvTorPath%\Tor\*.*" "%RsDeployPath%\tor" %Quite%
+	echo n | copy /-y "%EnvTorPath%\Tor\*.*" "%RsDeployPath%" %Quite%
 )
 
 rem pack files
@@ -221,8 +196,8 @@ endlocal
 exit /B 1
 
 :copy_extension
-if exist "%~1\lib\%~n1.dll" (
-	copy "%~1\lib\%~n1.dll" %2 %Quite%
+if exist "%~1\%RsBuildConfig%\%~n1.dll" (
+	copy "%~1\%RsBuildConfig%\%~n1.dll" %2 %Quite%
 )
 goto :EOF
 

build_scripts/Windows/env/env-msys2.bat

@@ -5,6 +5,7 @@
 call "%~dp0env.bat"
 if errorlevel 1 goto error_env
 
+rem openssl x86 doesn't compile with mingw64 x64
 :: Get gcc versions
 call "%ToolsPath%\get-gcc-version.bat" GCCVersion GCCArchitecture
 if "%GCCVersion%"=="" %cecho% error "Cannot get gcc version." & exit /B 1
@@ -22,12 +23,11 @@ if "%GCCArchitecture%"=="x64" (
 )
 
 set EnvMSYS2Path=%EnvRootPath%\msys2
-set EnvMSYS2BasePath=%EnvMSYS2Path%\msys64
 
 call "%~dp0tools\prepare-msys2.bat" %1
 if errorlevel 1 exit /B %ERRORLEVEL%
 
-set EnvMSYS2SH=%EnvMSYS2BasePath%\usr\bin\sh.exe
+set EnvMSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh.exe
 if not exist "%EnvMSYS2SH%" if errorlevel 1 goto error_env
 
 set EnvMSYS2Cmd="%EnvMSYS2SH%" -lc

build_scripts/Windows/env/env.bat

@@ -28,9 +28,6 @@ if errorlevel 1 exit /B %ERRORLEVEL%
 set PATH=%EnvToolsPath%\MinGit\cmd;%EnvToolsPath%\cmake\bin;%PATH%
 set HOME=%EnvToolsPath%\MinGit\home
 
-:: Add Doxygen to PATH
-set PATH=%EnvToolsPath%\doxygen;%PATH%
-
 exit /B 0
 
 :error_env

build_scripts/Windows/env/tools/prepare-msys2.bat

@@ -16,15 +16,7 @@ if "%~1"=="clean" (
 	goto exit
 )
 
-set MSYS2Version=20241208
-
-set MSYS2Install=msys2-base-x86_64-%MSYS2Version%.sfx.exe
-set MSYS2Url=https://github.com/msys2/msys2-installer/releases/download/%MSYS2Version:~0,4%-%MSYS2Version:~4,2%-%MSYS2Version:~6,2%/%MSYS2Install%
-set MSYS2UnpackPath=%EnvMSYS2Path%\msys64
-set CMakeInstall=cmake-3.31.3-windows-i386.zip
-set CMakeUrl=https://github.com/Kitware/CMake/releases/download/v3.31.3/%CMakeInstall%
-
-if exist "%MSYS2UnpackPath%\usr\bin\pacman.exe" (
+if exist "%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\pacman.exe" (
 	if "%~1"=="reinstall" (
 		choice /M "Found existing MSYS2 version. Do you want to proceed?"
 		if !ERRORLEVEL!==2 goto exit
@@ -33,12 +25,18 @@ if exist "%MSYS2UnpackPath%\usr\bin\pacman.exe" (
 	)
 )
 
-if exist "%MSYS2UnpackPath%" (
+set MSYS2Install=msys2-base-%MSYS2Architecture%-20190524.tar.xz
+set MSYS2Url=http://sourceforge.net/projects/msys2/files/Base/%MSYS2Architecture%/%MSYS2Install%/download
+set CMakeInstall=cmake-3.1.0-win32-x86.zip
+set CMakeUrl=http://www.cmake.org/files/v3.1/%CMakeInstall%
+set CMakeUnpackPath=%EnvMSYS2Path%\msys%MSYS2Base%
+
+if exist "%EnvMSYS2Path%\msys%MSYS2Base%" (
 	%cecho% info "Remove previous MSYS2 version"
-	call "%ToolsPath%\remove-dir.bat" "%MSYS2UnpackPath%"
+	call "%ToolsPath%\remove-dir.bat" "%EnvMSYS2Path%\msys%MSYS2Base%"
 )
 
-%cecho% info "Download MSYS2 installation files"
+%cecho% info "Download installation files"
 if not exist "%EnvDownloadPath%\%MSYS2Install%" call "%ToolsPath%\download-file.bat" "%MSYS2Url%" "%EnvDownloadPath%\%MSYS2Install%"
 if not exist "%EnvDownloadPath%\%MSYS2Install%" %cecho% error "Cannot download MSYS" & goto error
 
@@ -46,32 +44,37 @@ if not exist "%EnvDownloadPath%\%CMakeInstall%" call "%ToolsPath%\download-file.
 if not exist "%EnvDownloadPath%\%CMakeInstall%" %cecho% error "Cannot download CMake" & goto error
 
 %cecho% info "Unpack MSYS2"
-"%EnvDownloadPath%\%MSYS2Install%" -y -o"%EnvMSYS2Path%"
+"%EnvSevenZipExe%" x -so "%EnvDownloadPath%\%MSYS2Install%" | "%EnvSevenZipExe%" x -y -si -ttar -o"%EnvMSYS2Path%"
 
 %cecho% info "Unpack CMake"
-"%EnvSevenZipExe%" x -o"%MSYS2UnpackPath%" "%EnvDownloadPath%\%CMakeInstall%" -y -bso0
+"%EnvSevenZipExe%" x -o"%CMakeUnpackPath%" "%EnvDownloadPath%\%CMakeInstall%"
 
 %cecho% info "Install CMake"
 set CMakeVersion=
-for /D %%F in (%MSYS2UnpackPath%\cmake*) do set CMakeVersion=%%~nxF
+for /D %%F in (%CMakeUnpackPath%\cmake*) do set CMakeVersion=%%~nxF
 if "%CMakeVersion%"=="" %cecho% error "CMake version not found." & goto :exit
 %cecho% info "Found CMake version %CMakeVersion%"
 
 set FoundProfile=
-for /f "tokens=3" %%F in ('find /c /i "%CMakeVersion%" "%MSYS2UnpackPath%\etc\profile"') do set FoundProfile=%%F
+for /f "tokens=3" %%F in ('find /c /i "%CMakeVersion%" "%EnvMSYS2Path%\msys%MSYS2Base%\etc\profile"') do set FoundProfile=%%F
 
 if "%FoundProfile%"=="0" (
-	echo export PATH="${PATH}:/%CMakeVersion%/bin">>"%MSYS2UnpackPath%\etc\profile"
+	echo export PATH="${PATH}:/%CMakeVersion%/bin">>"%EnvMSYS2Path%\msys%MSYS2Base%\etc\profile"
 )
 
-set MSYS2SH=%MSYS2UnpackPath%\usr\bin\sh
+set MSYS2SH=%EnvMSYS2Path%\msys%MSYS2Base%\usr\bin\sh
+
+%cecho% info "Update keyring"
+"%MSYS2SH%" -lc "curl -O http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
+"%MSYS2SH%" -lc "pacman --noconfirm -U msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz"
 
 %cecho% info "Initialize MSYS2"
-"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
-"%MSYS2SH%" -lc "pacman --noconfirm -Sy"
-"%MSYS2SH%" -lc "pacman --noconfirm -Su"
+"%MSYS2SH%" -lc "pacman -Sy"
+"%MSYS2SH%" -lc "pacman --noconfirm --needed -S bash pacman pacman-mirrors msys2-runtime"
 
-call "%MSYS2UnpackPath%\autorebase.bat"
+call "%EnvMSYS2Path%\msys%MSYS2Base%\autorebase.bat"
+call "%EnvRootPath%\update-msys2.bat"
+call "%EnvRootPath%\update-msys2.bat"
 
 :exit
 endlocal

build_scripts/Windows/env/tools/prepare-tools.bat

@@ -16,12 +16,9 @@ set NSISInstallPath=%EnvToolsPath%\NSIS
 set MinGitInstall=MinGit-2.28.0-32-bit.zip
 set MinGitUrl=https://github.com/git-for-windows/git/releases/download/v2.28.0.windows.1/%MinGitInstall%
 set MinGitInstallPath=%EnvToolsPath%\MinGit
-set DoxygenInstall=doxygen-1.9.6.windows.x64.bin.zip
-set DoxygenUrl=https://github.com/doxygen/doxygen/releases/download/Release_1_9_6/%DoxygenInstall%
-set DoxygenInstallPath=%EnvToolsPath%\doxygen
-set CMakeVersion=cmake-3.31.3-windows-i386
+set CMakeVersion=cmake-3.1.0-win32-x86
 set CMakeInstall=%CMakeVersion%.zip
-set CMakeUrl=https://github.com/Kitware/CMake/releases/download/v3.31.3/%CMakeInstall%
+set CMakeUrl=http://www.cmake.org/files/v3.1/%CMakeInstall%
 set CMakeInstallPath=%EnvToolsPath%\cmake
 set TorProjectUrl=https://www.torproject.org
 set TorDownloadIndexUrl=%TorProjectUrl%/download/tor
@@ -53,7 +50,7 @@ if not exist "%EnvToolsPath%\cecho.exe" (
 	if not exist "%EnvDownloadPath%\%cCEhoInstall%" echo Cannot download cecho installation& goto error
 
 	echo Unpack cecho
-	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%CEchoInstall%" -y -bso0
+	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%CEchoInstall%"
 	copy "%EnvTempPath%\cecho.exe" "%EnvToolsPath%"
 
 	call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%"
@@ -69,7 +66,7 @@ if not exist "%EnvToolsPath%\depends.exe" (
 	if not exist "%EnvDownloadPath%\%DependsInstall%" %cecho% error "Cannot download Dependendy Walker installation" & goto error
 
 	%cecho% info "Unpack Dependency Walker"
-	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%DependsInstall%" -y -bso0
+	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%DependsInstall%"
 	copy "%EnvTempPath%\*" "%EnvToolsPath%"
 
 	call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%"
@@ -85,7 +82,7 @@ if not exist "%EnvToolsPath%\cut.exe" (
 	if not exist "%EnvDownloadPath%\%UnixToolsInstall%" %cecho% error "Cannot download Unix Tools installation" & goto error
 
 	%cecho% info "Unpack Unix Tools"
-	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%UnixToolsInstall%" -y -bso0
+	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%UnixToolsInstall%"
 	copy "%EnvTempPath%\cut.exe" "%EnvToolsPath%"
 
 	call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%"
@@ -101,7 +98,7 @@ if not exist "%EnvToolsPath%\sed.exe" (
 	if not exist "%EnvDownloadPath%\%UnixToolsInstall%" %cecho% error "Cannot download Unix Tools installation" & goto error
 
 	%cecho% info "Unpack Unix Tools"
-	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%UnixToolsInstall%" -y -bso0
+	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%UnixToolsInstall%"
 	copy "%EnvTempPath%\sed.exe" "%EnvToolsPath%"
 
 	call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%"
@@ -121,7 +118,7 @@ if not exist "%NSISInstallPath%\nsis.exe" (
 	if not exist "%EnvDownloadPath%\%NSISInstall%" %cecho% error "Cannot download NSIS installation" & goto error
 
 	%cecho% info "Unpack NSIS"
-	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%NSISInstall%" -y -bso0
+	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%NSISInstall%"
 	if not exist "%NSISInstallPath%" mkdir "%NSISInstallPath%"
 	xcopy /s "%EnvTempPath%" "%NSISInstallPath%"
 
@@ -135,20 +132,7 @@ if not exist "%MinGitInstallPath%\cmd\git.exe" (
 	if not exist "%EnvDownloadPath%\%MinGitInstall%" %cecho% error "Cannot download MinGit installation" & goto error
 
 	%cecho% info "Unpack MinGit"
-	"%EnvSevenZipExe%" x -o"%MinGitInstallPath%" "%EnvDownloadPath%\%MinGitInstall%" -y -bso0
-)
-
-if not exist "%EnvDownloadPath%\%DoxygenInstall%" call "%ToolsPath%\remove-dir.bat" "%DoxygenInstallPath%"
-if not exist "%DoxygenInstallPath%\doxygen.exe" (
-	if exist "%DoxygenInstallPath%" call "%ToolsPath%\remove-dir.bat" "%DoxygenInstallPath%"
-
-	%cecho% info "Download Doxygen installation"
-
-	if not exist "%EnvDownloadPath%\%DoxygenInstall%" call "%ToolsPath%\download-file.bat" "%DoxygenUrl%" "%EnvDownloadPath%\%DoxygenInstall%"
-	if not exist "%EnvDownloadPath%\%DoxygenInstall%" %cecho% error "Cannot download doxygen installation" & goto error
-
-	%cecho% info "Unpack Doxygen"
-	"%EnvSevenZipExe%" x -o"%DoxygenInstallPath%" "%EnvDownloadPath%\%DoxygenInstall%" -y -bso0
+	"%EnvSevenZipExe%" x -o"%MinGitInstallPath%" "%EnvDownloadPath%\%MinGitInstall%"
 )
 
 if not exist "%EnvDownloadPath%\%CMakeInstall%" call "%ToolsPath%\remove-dir.bat" "%CMakeInstallPath%"
@@ -163,7 +147,7 @@ if not exist "%CMakeInstallPath%\bin\cmake.exe" (
 	if not exist "%EnvDownloadPath%\%CMakeInstall%" %cecho% error "Cannot download CMake installation" & goto error
 
 	%cecho% info "Unpack CMake"
-	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%CMakeInstall%" -y -bso0
+	"%EnvSevenZipExe%" x -o"%EnvTempPath%" "%EnvDownloadPath%\%CMakeInstall%"
 
 	move "%EnvTempPath%\%CMakeVersion%" "%CMakeInstallPath%"
 
@@ -176,7 +160,7 @@ mkdir "%EnvTempPath%"
 call "%ToolsPath%\download-file.bat" "%TorDownloadIndexUrl%" "%EnvTempPath%\index.html"
 if not exist "%EnvTempPath%\index.html" %cecho% error "Cannot download Tor installation" & goto error
 
-for /F "tokens=1,2 delims= " %%A in ('%EnvSedExe% -r -n -e"s/.*href=\"^(.*^)^(tor-.*windows-i686.*\.tar\.gz^)\".*/\2 \1\2/p" "%EnvTempPath%\index.html"') do set TorInstall=%%A& set TorDownloadUrl=%%B
+for /F "tokens=1,2 delims= " %%A in ('%EnvSedExe% -r -n -e"s/.*href=\"^(.*^)^(tor-win32.*\.zip^)\".*/\2 \1\2/p" "%EnvTempPath%\index.html"') do set TorInstall=%%A& set TorDownloadUrl=%TorProjectUrl%%%B
 call "%ToolsPath%\remove-dir.bat" "%EnvTempPath%"
 if "%TorInstall%"=="" %cecho% error "Cannot download Tor installation" & goto error
 if "%TorDownloadUrl%"=="" %cecho% error "Cannot download Tor installation" & goto error
@@ -189,7 +173,7 @@ if not exist "%EnvTorPath%\Tor\tor.exe" (
 	if not exist "%EnvDownloadPath%\%TorInstall%" %cecho% error "Cannot download Tor installation" & goto error
 
 	%cecho% info "Unpack Tor"
-	"%EnvSevenZipExe%" x -so "%EnvDownloadPath%\%TorInstall%" | "%EnvSevenZipExe%" x -si -ttar -o"%EnvTorPath%" -y -bso0
+	"%EnvSevenZipExe%" x -o"%EnvTorPath%" "%EnvDownloadPath%\%TorInstall%"
 )
 
 :exit

build_scripts/Windows/env/tools/root/update-msys2.bat

@@ -2,13 +2,18 @@
 
 setlocal
 
-if not exist "%~dp0msys2\msys64" goto :EOF
+if exist "%~dp0msys2\msys32" call :update 32
+if exist "%~dp0msys2\msys64" call :update 64
 
-set MSYS2SH=%~dp0msys2\msys64\usr\bin\sh
+goto :EOF
 
-echo Update MSYS2
-"%MSYS2SH%" -lc "yes | pacman --noconfirm -Syuu msys2-keyring"
-"%MSYS2SH%" -lc "pacman --noconfirm -Su"
+:update
+set MSYSSH=%~dp0msys2\msys%~1\usr\bin\sh
 
+echo Update MSYS2 %~1
+"%MSYSSH%" -lc "pacman -Sy"
+"%MSYSSH%" -lc "pacman --noconfirm -Su"
+
+:exit
 endlocal
 goto :EOF

build_scripts/Windows/installer/lang/ca_ES.nsh

@@ -2,12 +2,6 @@
 !insertmacro LANG_STRING Section_Main_Desc "Instal·la ${APPNAME} i els components necessaris."
 !insertmacro LANG_STRING Section_Tor "Tor"
 !insertmacro LANG_STRING Section_Tor_Desc "Installs Tor."
-!insertmacro LANG_STRING Section_WebUI "WebUI"
-!insertmacro LANG_STRING Section_WebUI_Desc "Installs WebUI."
-!insertmacro LANG_STRING Section_Service "Service"
-!insertmacro LANG_STRING Section_Service_Desc "Installs Service."
-!insertmacro LANG_STRING Section_FriendServer "Friend Server"
-!insertmacro LANG_STRING Section_FriendServer_Desc "Installs Friend Server."
 !insertmacro LANG_STRING Section_Data "Pells"
 !insertmacro LANG_STRING Section_Data_Desc "Instal·la pells."
 !insertmacro LANG_STRING Section_Shortcuts "Icones d'accés directe"

build_scripts/Windows/installer/lang/de.nsh

@@ -2,12 +2,6 @@
 !insertmacro LANG_STRING Section_Main_Desc "Installiert ${APPNAME} und die benötigten Komponenten."
 !insertmacro LANG_STRING Section_Tor "Tor"
 !insertmacro LANG_STRING Section_Tor_Desc "Installiert Tor."
-!insertmacro LANG_STRING Section_WebUI "WebUI"
-!insertmacro LANG_STRING Section_WebUI_Desc "Installiert WebUI."
-!insertmacro LANG_STRING Section_Service "Service"
-!insertmacro LANG_STRING Section_Service_Desc "Installiert Service."
-!insertmacro LANG_STRING Section_FriendServer "Friend Server"
-!insertmacro LANG_STRING Section_FriendServer_Desc "Installiert Friend Server."
 !insertmacro LANG_STRING Section_Data "Skins"
 !insertmacro LANG_STRING Section_Data_Desc "Skins installieren."
 !insertmacro LANG_STRING Section_Shortcuts "Verknüpfungssymbole"

build_scripts/Windows/installer/lang/en.nsh

@@ -2,12 +2,6 @@
 !insertmacro LANG_STRING Section_Main_Desc "Installs ${APPNAME} and required components."
 !insertmacro LANG_STRING Section_Tor "Tor"
 !insertmacro LANG_STRING Section_Tor_Desc "Installs Tor."
-!insertmacro LANG_STRING Section_WebUI "WebUI"
-!insertmacro LANG_STRING Section_WebUI_Desc "Installs WebUI."
-!insertmacro LANG_STRING Section_Service "Service"
-!insertmacro LANG_STRING Section_Service_Desc "Installs Service."
-!insertmacro LANG_STRING Section_FriendServer "Friend Server"
-!insertmacro LANG_STRING Section_FriendServer_Desc "Installs Friend Server."
 !insertmacro LANG_STRING Section_Data "Skins"
 !insertmacro LANG_STRING Section_Data_Desc "Installs skins."
 !insertmacro LANG_STRING Section_Shortcuts "Shortcut icons"

build_scripts/Windows/installer/lang/es.nsh

@@ -2,12 +2,6 @@
 !insertmacro LANG_STRING Section_Main_Desc "Instala ${APPNAME} y los componentes requeridos."
 !insertmacro LANG_STRING Section_Tor "Tor"
 !insertmacro LANG_STRING Section_Tor_Desc "Installs Tor."
-!insertmacro LANG_STRING Section_WebUI "WebUI"
-!insertmacro LANG_STRING Section_WebUI_Desc "Installs WebUI."
-!insertmacro LANG_STRING Section_Service "Service"
-!insertmacro LANG_STRING Section_Service_Desc "Installs Service."
-!insertmacro LANG_STRING Section_FriendServer "Friend Server"
-!insertmacro LANG_STRING Section_FriendServer_Desc "Installs Friend Server."
 !insertmacro LANG_STRING Section_Data "Coberturas (skins)"
 !insertmacro LANG_STRING Section_Data_Desc "Instalar coberturas"
 !insertmacro LANG_STRING Section_Shortcuts "Iconos de accesos directos"

build_scripts/Windows/installer/lang/fr.nsh

@@ -2,12 +2,6 @@
 !insertmacro LANG_STRING Section_Main_Desc "Installe ${APPNAME} et les composants requis."
 !insertmacro LANG_STRING Section_Tor "Tor"
 !insertmacro LANG_STRING Section_Tor_Desc "Installs Tor."
-!insertmacro LANG_STRING Section_WebUI "WebUI"
-!insertmacro LANG_STRING Section_WebUI_Desc "Installs WebUI."
-!insertmacro LANG_STRING Section_Service "Service"
-!insertmacro LANG_STRING Section_Service_Desc "Installs Service."
-!insertmacro LANG_STRING Section_FriendServer "Friend Server"
-!insertmacro LANG_STRING Section_FriendServer_Desc "Installs Friend Server."
 !insertmacro LANG_STRING Section_Data "Habillages"
 !insertmacro LANG_STRING Section_Data_Desc "Installe des habillages."
 !insertmacro LANG_STRING Section_Shortcuts "Icônes de raccourci"

build_scripts/Windows/installer/lang/pl.nsh

@@ -2,12 +2,6 @@
 !insertmacro LANG_STRING Section_Main_Desc "Instaluje ${APPNAME} oraz wymagane komponenty."
 !insertmacro LANG_STRING Section_Tor "Tor"
 !insertmacro LANG_STRING Section_Tor_Desc "Installs Tor."
-!insertmacro LANG_STRING Section_WebUI "WebUI"
-!insertmacro LANG_STRING Section_WebUI_Desc "Installs WebUI."
-!insertmacro LANG_STRING Section_Service "Service"
-!insertmacro LANG_STRING Section_Service_Desc "Installs Service."
-!insertmacro LANG_STRING Section_FriendServer "Friend Server"
-!insertmacro LANG_STRING Section_FriendServer_Desc "Installs Friend Server."
 !insertmacro LANG_STRING Section_Data "Skórki"
 !insertmacro LANG_STRING Section_Data_Desc "Instaluje skórki."
 !insertmacro LANG_STRING Section_Shortcuts "Ikony skrótów"

build_scripts/Windows/installer/lang/ru.nsh

@@ -2,12 +2,6 @@
 !insertmacro LANG_STRING Section_Main_Desc "Установка ${APPNAME} и необходимых компонентов."
 !insertmacro LANG_STRING Section_Tor "Tor"
 !insertmacro LANG_STRING Section_Tor_Desc "Installs Tor."
-!insertmacro LANG_STRING Section_WebUI "WebUI"
-!insertmacro LANG_STRING Section_WebUI_Desc "Installs WebUI."
-!insertmacro LANG_STRING Section_Service "Service"
-!insertmacro LANG_STRING Section_Service_Desc "Installs Service."
-!insertmacro LANG_STRING Section_FriendServer "Friend Server"
-!insertmacro LANG_STRING Section_FriendServer_Desc "Installs Friend Server."
 !insertmacro LANG_STRING Section_Data "Оболочки"
 !insertmacro LANG_STRING Section_Data_Desc "Установка оболочек."
 !insertmacro LANG_STRING Section_Shortcuts "Ярлыки"

build_scripts/Windows/installer/lang/tr.nsh

@@ -2,12 +2,6 @@
 !insertmacro LANG_STRING Section_Main_Desc "${APPNAME} ve gerekli bileşenleri kurar."
 !insertmacro LANG_STRING Section_Tor "Tor"
 !insertmacro LANG_STRING Section_Tor_Desc "Installs Tor."
-!insertmacro LANG_STRING Section_WebUI "WebUI"
-!insertmacro LANG_STRING Section_WebUI_Desc "Installs WebUI."
-!insertmacro LANG_STRING Section_Service "Service"
-!insertmacro LANG_STRING Section_Service_Desc "Installs Service."
-!insertmacro LANG_STRING Section_FriendServer "Friend Server"
-!insertmacro LANG_STRING Section_FriendServer_Desc "Installs Friend Server."
 !insertmacro LANG_STRING Section_Data "Temalar"
 !insertmacro LANG_STRING Section_Data_Desc "Tema yükleyin."
 !insertmacro LANG_STRING Section_Shortcuts "Kısayol simgeleri"

build_scripts/Windows/installer/lang/ts/en.ts

@@ -22,48 +22,6 @@
     <translation type="unfinished"></translation> 
   </message> 
 </context> 
-<context> 
-  <name>Section_WebUI</name> 
-  <message> 
-    <source>WebUI</source> 
-    <translation type="unfinished"></translation> 
-  </message> 
-</context> 
-<context> 
-  <name>Section_WebUI_Desc</name> 
-  <message> 
-    <source>Installs WebUI.</source> 
-    <translation type="unfinished"></translation> 
-  </message> 
-</context> 
-<context> 
-  <name>Section_Service</name> 
-  <message> 
-    <source>Service</source> 
-    <translation type="unfinished"></translation> 
-  </message> 
-</context> 
-<context> 
-  <name>Section_Service_Desc</name> 
-  <message> 
-    <source>Installs Service.</source> 
-    <translation type="unfinished"></translation> 
-  </message> 
-</context> 
-<context> 
-  <name>Section_FriendServer</name> 
-  <message> 
-    <source>Friend Server</source> 
-    <translation type="unfinished"></translation> 
-  </message> 
-</context> 
-<context> 
-  <name>Section_FriendServer_Desc</name> 
-  <message> 
-    <source>Installs Friend Server.</source> 
-    <translation type="unfinished"></translation> 
-  </message> 
-</context> 
 <context> 
   <name>Section_Data</name> 
   <message> 

build_scripts/Windows/installer/lang/zh_CN.nsh

@@ -2,12 +2,6 @@
 !insertmacro LANG_STRING Section_Main_Desc "Installs ${APPNAME} and required components."
 !insertmacro LANG_STRING Section_Tor "Tor"
 !insertmacro LANG_STRING Section_Tor_Desc "Installs Tor."
-!insertmacro LANG_STRING Section_WebUI "WebUI"
-!insertmacro LANG_STRING Section_WebUI_Desc "Installs WebUI."
-!insertmacro LANG_STRING Section_Service "Service"
-!insertmacro LANG_STRING Section_Service_Desc "Installs Service."
-!insertmacro LANG_STRING Section_FriendServer "Friend Server"
-!insertmacro LANG_STRING Section_FriendServer_Desc "Installs Friend Server."
 !insertmacro LANG_STRING Section_Data "皮肤"
 !insertmacro LANG_STRING Section_Data_Desc "安装皮肤"
 !insertmacro LANG_STRING Section_Shortcuts "快捷方式图标"

build_scripts/Windows/installer/retroshare-Qt5.nsi

@@ -60,9 +60,6 @@
 !define /date DATE "%Y%m%d"
 !endif
 
-# Service
-${!defineifexist} SERVICE_EXISTS "${RELEASEDIR}\retroshare-service\src\release\retroshare-service.exe"
-
 # Tor
 !ifdef TORDIR
 ${!defineifexist} TOR_EXISTS "${TORDIR}\tor.exe"
@@ -71,20 +68,6 @@ ${!defineifexist} TOR_EXISTS "${TORDIR}\tor.exe"
 !endif
 !endif
 
-# WebUI
-!ifdef WEBUIDIR
-${!defineifexist} WEBUI_EXISTS "${WEBUIDIR}\index.html"
-!ifndef WEBUI_EXISTS
-!error "WebUI files not found"
-!endif
-!endif
-
-# Friend Server
-!ifdef TOR_EXISTS
-# Add Friend Server with Tor only
-#${!defineifexist} FRIENDSERVER_EXISTS "${RELEASEDIR}\retroshare-friendserver\src\release\retroshare-friendserver.exe"
-!endif
-
 # Application name and version
 !define APPNAME "RetroShare"
 !define APPNAMEANDVERSION "${APPNAME} ${VERSION}"
@@ -209,8 +192,8 @@ Section $(Section_Main) Section_Main
 
   ; Main binaries
   SetOutPath "$INSTDIR"
-  File "${RELEASEDIR}\retroshare-gui\src\release\retroshare.exe"
-  File /nonfatal "${RELEASEDIR}\libretroshare\src\lib\retroshare.dll"
+  File /oname=retroshare.exe "${RELEASEDIR}\retroshare-gui\src\release\retroshare.exe"
+  File /oname=retroshare-service.exe "${RELEASEDIR}\retroshare-service\src\release\retroshare-service.exe"
 
   ; Qt binaries
   File "${QTDIR}\bin\Qt5Core.dll"
@@ -247,7 +230,6 @@ Section $(Section_Main) Section_Main
 
   ; External binaries
   File "${EXTERNAL_LIB_DIR}\bin\miniupnpc.dll"
-  File "${RELEASEDIR}\supportlibs\librnp\Build\src\lib\librnp.dll"
   !if ${ARCHITECTURE} == "x86"
     File "${EXTERNAL_LIB_DIR}\bin\libcrypto-1_1.dll"
     File "${EXTERNAL_LIB_DIR}\bin\libssl-1_1.dll"
@@ -299,41 +281,17 @@ Section $(Section_Main) Section_Main
   File /r "${SOURCEDIR}\retroshare-gui\src\license\*.*"
 SectionEnd
 
-# Service
-!ifdef SERVICE_EXISTS
-  Section /o $(Section_Service) Section_Service
-    SetOutPath "$INSTDIR"
-  File "${RELEASEDIR}\retroshare-service\src\release\retroshare-service.exe"
-  SectionEnd
-!endif
-
-# Friend Server
-!ifdef FRIENDSERVER_EXISTS
-  Section /o $(Section_FriendServer) Section_FriendServer
-    SetOutPath "$INSTDIR"
-    File "${RELEASEDIR}\retroshare-friendserver\src\release\retroshare-friendserver.exe"
-  SectionEnd
-!endif
-
 # Tor
 !ifdef TOR_EXISTS
   Section /o $(Section_Tor) Section_Tor
-    SetOutPath "$INSTDIR\tor"
-    File "${TORDIR}\*"
-  SectionEnd
-!endif
-
-# WebUI
-!ifdef WEBUI_EXISTS
-  Section /o $(Section_WebUI) Section_WebUI
-    SetOutPath "$INSTDIR\webui"
-    File /r "${WEBUIDIR}\*"
+    SetOutPath "$INSTDIR"
+    File /r "${TORDIR}\*"
   SectionEnd
 !endif
 
 # Plugins
-${!defineifexist} PLUGIN_FEEDREADER_EXISTS "${RELEASEDIR}\plugins\FeedReader\lib\FeedReader.dll"
-${!defineifexist} PLUGIN_VOIP_EXISTS "${RELEASEDIR}\plugins\VOIP\lib\VOIP.dll"
+${!defineifexist} PLUGIN_FEEDREADER_EXISTS "${RELEASEDIR}\plugins\FeedReader\release\FeedReader.dll"
+${!defineifexist} PLUGIN_VOIP_EXISTS "${RELEASEDIR}\plugins\VOIP\release\VOIP.dll"
 
 !ifdef PLUGIN_FEEDREADER_EXISTS
 !define /ifndef PLUGIN_EXISTS
@@ -347,16 +305,14 @@ ${!defineifexist} PLUGIN_VOIP_EXISTS "${RELEASEDIR}\plugins\VOIP\lib\VOIP.dll"
   !ifdef PLUGIN_FEEDREADER_EXISTS
     Section $(Section_Plugin_FeedReader) Section_Plugin_FeedReader
       SetOutPath "$DataDir\extensions6"
-      File "${RELEASEDIR}\plugins\FeedReader\lib\FeedReader.dll"
+      File "${RELEASEDIR}\plugins\FeedReader\release\FeedReader.dll"
     SectionEnd
   !endif
 
   !ifdef PLUGIN_VOIP_EXISTS
     Section $(Section_Plugin_VOIP) Section_Plugin_VOIP
       SetOutPath "$DataDir\extensions6"
-      File "${RELEASEDIR}\plugins\VOIP\lib\VOIP.dll"
-      SetOutPath "$INSTDIR\sounds"
-      File /r "${SOURCEDIR}\plugins\VOIP\gui\sounds\*.*"
+      File "${RELEASEDIR}\plugins\VOIP\release\VOIP.dll"
     SectionEnd
   !endif
   SectionGroupEnd
@@ -375,7 +331,7 @@ Section $(Section_Data) Section_Data
 
   ; Stylesheets
   SetOutPath "$INSTDIR\qss"
-  File /nonfatal /r "${SOURCEDIR}\retroshare-gui\src\qss\*.*"
+  File /r "${SOURCEDIR}\retroshare-gui\src\qss\*.*"
 SectionEnd
 
 ;Section $(Section_Link) Section_Link
@@ -396,22 +352,6 @@ Section $(Section_StartMenu) Section_StartMenu
   CreateDirectory "$SMPROGRAMS\${APPNAME}"
   CreateShortCut "$SMPROGRAMS\${APPNAME}\$(Link_Uninstall).lnk" "$INSTDIR\uninstall.exe" "" "$INSTDIR\uninstall.exe" 0
   CreateShortCut "$SMPROGRAMS\${APPNAME}\${APPNAME}.lnk" "$INSTDIR\retroshare.exe" "" "$INSTDIR\retroshare.exe" 0
-
-!ifdef SERVICE_EXISTS
-  SectionGetFlags ${Section_Service} $0
-  IntOp $0 $0 & ${SF_SELECTED}
-  ${If} $0 == ${SF_SELECTED}
-    CreateShortCut "$SMPROGRAMS\${APPNAME}\${APPNAME} Service.lnk" "$INSTDIR\retroshare-service.exe" "" "$INSTDIR\retroshare-service.exe" 0
-  ${EndIf}
-!endif
-
-!ifdef FRIENDSERVER_EXISTS
-  SectionGetFlags ${Section_FriendServer} $0
-  IntOp $0 $0 & ${SF_SELECTED}
-  ${If} $0 == ${SF_SELECTED}
-    CreateShortCut "$SMPROGRAMS\${APPNAME}\${APPNAME} Friend Server.lnk" "$INSTDIR\retroshare-friendserver.exe" "" "$INSTDIR\retroshare-friendserver.exe" 0
-  ${EndIf}
-!endif
 SectionEnd
 
 Section $(Section_Desktop) Section_Desktop
@@ -464,9 +404,6 @@ SectionEnd
 ;  !insertmacro MUI_DESCRIPTION_TEXT ${Section_Link} $(Section_Link_Desc)
   !insertmacro MUI_DESCRIPTION_TEXT ${Section_AutoStart} $(Section_AutoStart_Desc)
   !insertmacro MUI_DESCRIPTION_TEXT ${Section_Tor} $(Section_Tor_Desc)
-  !insertmacro MUI_DESCRIPTION_TEXT ${Section_WebUI} $(Section_WebUI_Desc)
-  !insertmacro MUI_DESCRIPTION_TEXT ${Section_Service} $(Section_Service_Desc)
-  !insertmacro MUI_DESCRIPTION_TEXT ${Section_FriendServer} $(Section_FriendServer_Desc)
 !insertmacro MUI_FUNCTION_DESCRIPTION_END
 
 # Uninstall
@@ -521,26 +458,6 @@ Function .onInit
   !insertmacro MUI_LANGDLL_DISPLAY
 FunctionEnd
 
-!ifdef FRIENDSERVER_EXISTS
-Function .onSelChange
-  SectionGetFlags ${Section_FriendServer} $0
-  IntOp $0 $0 & ${SF_SELECTED}
-  ${If} $0 == ${SF_SELECTED}
-    # Activate Tor and set readonly
-    SectionGetFlags ${Section_Tor} $1
-    IntOp $1 $1 | ${SF_SELECTED}
-    IntOp $1 $1 | ${SF_RO}
-    SectionSetFlags ${Section_Tor} $1
-  ${Else}
-    # Remove readonly from Tor
-    SectionGetFlags ${Section_Tor} $1
-    IntOp $2 ${SF_RO} ~
-    IntOp $1 $1 & $2
-    SectionSetFlags ${Section_Tor} $1
-  ${EndIf}
-FunctionEnd
-!endif
-
 # Installation mode
 
 Function RequireAdmin

build_scripts/Windows/tools/generate-changelog.bat

@@ -18,8 +18,6 @@ copy nul %logfile% > nul
 
 pushd %~1
 
-set Percent=%%
-
 set last=HEAD
 for /f %%t in ('git tag --sort=-taggerdate --merged ^| findstr v') do (
 	echo generating changelog for !last!..%%t	
@@ -32,7 +30,7 @@ for /f %%t in ('git tag --sort=-taggerdate --merged ^| findstr v') do (
 	rem echo !last! ---^> %%t >> %logfile%
 	echo ----------------------------------------------- >> %logfile%
 	echo. >> %logfile%
-	git log %%t..!last! --no-merges "--pretty=format:!Percent!h !Percent!ai !Percent!<(10,trunc)!Percent!an !Percent!s" >> %logfile%
+	git log %%t..!last! --no-merges "--pretty=format:%%h %%ai %%<(10,trunc)%%an %%s" >> %logfile%
 	echo. >> %logfile%
 	echo. >> %logfile%
 	set last=%%t

build_scripts/git_tag_cleaner.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-function git_del_tag()
-{
-	mTag=$1
-
-	for mRemote in $(git remote); do
-		echo "Attempting tag $mTag removal from remote $mRemote"
-		GIT_TERMINAL_PROMPT=0 git push $mRemote :$mTag || true
-	done
-	git tag --delete $mTag
-}
-
-for mModule in . build_scripts/OBS/ libbitdht/ libretroshare/ openpgpsdk/ retroshare-webui/ ; do
-	pushd $mModule
-	git_del_tag v0.6.7a
-	git tag --list | grep untagged | while read mTag; do git_del_tag $mTag ; done
-	popd
-done
-

data/retroshare.desktop

@@ -7,5 +7,5 @@ Exec=/usr/bin/retroshare %U
 Icon=/usr/share/pixmaps/retroshare.xpm
 Terminal=false
 Type=Application
-Categories=Application;Network;Email;InstantMessaging;Chat;Feed;FileTransfer;P2P
+Categories=Application;Network;P2P;Feed;Chat;InstantMessaging
 MimeType=x-scheme-handler/retroshare;

jsonapi-generator/README.adoc

@@ -1,10 +1,389 @@
-// SPDX-FileCopyrightText: (C) 2021 Retroshare Team <contact@retroshare.cc>
+// SPDX-FileCopyrightText: (C) 2004-2019 Retroshare Team <contact@retroshare.cc>
 // SPDX-License-Identifier: CC0-1.0
 
-= Moved
+RetroShare JSON API
+===================
 
-JSON API generator is now part of `libretroshare` look under `src/jsonapi/`
-directory in `libretroshare` repository.
+:Cxx: C&#43;&#43;
 
-This directory and all it's content is kept as is only for retro-compatibility
-with the old, deprecated `qmake` build system.
+== How to use RetroShare JSON API
+
+Look for methods marked with +@jsonapi+ doxygen custom command into
++libretroshare/src/retroshare+. The method path is composed by service instance
+pointer name like +rsGxsChannels+ for +RsGxsChannels+, and the method name like
++createGroup+ and pass the input paramethers as a JSON object.
+
+.Service instance pointer in rsgxschannels.h
+[source,cpp]
+--------------------------------------------------------------------------------
+/**
+ * Pointer to global instance of RsGxsChannels service implementation
+ * @jsonapi{development}
+ */
+extern RsGxsChannels* rsGxsChannels;
+--------------------------------------------------------------------------------
+
+.Method declaration in rsgxschannels.h
+[source,cpp]
+--------------------------------------------------------------------------------
+	/**
+	 * @brief Request channel creation.
+	 * The action is performed asyncronously, so it could fail in a subsequent
+	 * phase even after returning true.
+	 * @jsonapi{development}
+	 * @param[out] token Storage for RsTokenService token to track request
+	 * status.
+	 * @param[in] group Channel data (name, description...)
+	 * @return false on error, true otherwise
+	 */
+	virtual bool createGroup(uint32_t& token, RsGxsChannelGroup& group) = 0;
+--------------------------------------------------------------------------------
+
+.paramethers.json
+[source,json]
+--------------------------------------------------------------------------------
+{
+    "group":{
+        "mMeta":{
+            "mGroupName":"JSON test group",
+            "mGroupFlags":4,
+            "mSignFlags":520
+        },
+        "mDescription":"JSON test group description"
+    },
+    "caller_data":"Here can go any kind of JSON data (even objects) that the caller want to get back together with the response"
+}
+--------------------------------------------------------------------------------
+
+.Calling the JSON API with curl on the terminal
+[source,bash]
+--------------------------------------------------------------------------------
+curl -u $API_USER --data @paramethers.json http://127.0.0.1:9092/rsGxsChannels/createGroup
+--------------------------------------------------------------------------------
+
+.JSON API call result
+[source,json]
+--------------------------------------------------------------------------------
+{
+    "caller_data": "Here can go any kind of JSON data (even objects) that the caller want to get back together with the response",
+    "retval": true,
+    "token": 3
+}
+--------------------------------------------------------------------------------
+
+Even if it is less efficient because of URL encoding HTTP +GET+ method is
+supported too, so in cases where the client cannot use +POST+ she can still use
++GET+ taking care of encoding the JSON data. With +curl+ this can be done at
+least in two different ways.
+
+.Calling the JSON API with GET method with curl on the terminal
+[source,bash]
+--------------------------------------------------------------------------------
+curl -u $API_USER --get --data-urlencode jsonData@paramethers.json \
+	http://127.0.0.1:9092/rsGxsChannels/createGroup
+--------------------------------------------------------------------------------
+
+Letting +curl+ do the encoding is much more elegant but it is semantically
+equivalent to the following.
+
+.Calling the JSON API with GET method and pre-encoded data with curl on the terminal
+--------------------------------------------------------------------------------
+curl -u $API_USER http://127.0.0.1:9092/rsGxsChannels/createGroup?jsonData=%7B%0A%20%20%20%20%22group%22%3A%7B%0A%20%20%20%20%20%20%20%20%22mMeta%22%3A%7B%0A%20%20%20%20%20%20%20%20%20%20%20%20%22mGroupName%22%3A%22JSON%20test%20group%22%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%22mGroupFlags%22%3A4%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%22mSignFlags%22%3A520%0A%20%20%20%20%20%20%20%20%7D%2C%0A%20%20%20%20%20%20%20%20%22mDescription%22%3A%22JSON%20test%20group%20description%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22caller_data%22%3A%22Here%20can%20go%20any%20kind%20of%20JSON%20data%20%28even%20objects%29%20that%20the%20caller%20want%20to%20get%20back%20together%20with%20the%20response%22%0A%7D
+--------------------------------------------------------------------------------
+
+Note that using +GET+ method +?jsonData=+ and then the JSON data URL encoded are
+added after the path in the HTTP request.
+
+
+== JSON API authentication
+
+Most of JSON API methods require authentication as they give access to
+RetroShare user data, and we don't want any application running on the system
+eventually by other users be able to access private data indiscriminately.
+JSON API support HTTP Basic as authentication scheme, this is enough as JSON API
+server is intented for usage on the same system (127.0.0.1) not over an
+untrusted network.
+If you need to use JSON API over an untrusted network consider using a reverse
+proxy with HTTPS such as NGINX in front of JSON API server.
+If RetroShare login has been effectuated through the JSON API you can use your
+location SSLID as username and your PGP password as credential for the JSON API,
+but we suggests you use specific meaningful and human readable credentials for
+each JSON API client so the human user can have better control over which client
+can access the JSON API.
+
+.NewToken.json
+[source,json]
+--------------------------------------------------------------------------------
+{
+	"token": "myNewUser:myNewPassword"
+}
+--------------------------------------------------------------------------------
+
+.An authenticated client can authorize new tokens like this
+--------------------------------------------------------------------------------
+curl -u $API_USER --data @NewToken.json http://127.0.0.1:9092/jsonApiServer/authorizeToken
+--------------------------------------------------------------------------------
+
+.An unauthenticated JSON API client can request access with
+--------------------------------------------------------------------------------
+curl --data @NewToken.json http://127.0.0.1:9092/jsonApiServer/requestNewTokenAutorization
+--------------------------------------------------------------------------------
+
+When an unauthenticated client request his token to be authorized, JSON API
+server will try to ask confirmation to the human user if possible through
++mNewAccessRequestCallback+, if it is not possible or the user didn't authorized
+the token +false+ is returned.
+
+
+== Offer new RetroShare services through JSON API
+
+To offer a retroshare service through the JSON API, first of all one need find
+the global pointer to the service instance and document it in doxygen syntax,
+plus marking with the custom doxygen command +@jsonapi{RS_VERSION}+ where
++RS_VERSION+ is the retroshare version in which this service became available
+with the current semantic (major changes to the service semantic, changes the
+meaning of the service itself, so the version should be updated in the
+documentation in that case).
+
+.Service instance pointer in rsgxschannels.h
+[source,cpp]
+--------------------------------------------------------------------------------
+/**
+ * Pointer to global instance of RsGxsChannels service implementation
+ * @jsonapi{development}
+ */
+extern RsGxsChannels* rsGxsChannels;
+--------------------------------------------------------------------------------
+
+
+Once the service instance itself is known to the JSON API you need to document
+in doxygen syntax and mark with the custom doxygen command
++@jsonapi{RS_VERSION}+ the methods of the service that you want to make
+available through JSON API.
+
+.Offering RsGxsChannels::getChannelDownloadDirectory in rsgxschannels.h
+[source,cpp]
+--------------------------------------------------------------------------------
+	/**
+	 * Get download directory for the given channel
+	 * @jsonapi{development}
+	 * @param[in] channelId id of the channel
+	 * @param[out] directory reference to string where to store the path
+	 * @return false on error, true otherwise
+	 */
+	virtual bool getChannelDownloadDirectory( const RsGxsGroupId& channelId,
+	                                          std::string& directory ) = 0;
+--------------------------------------------------------------------------------
+
+For each paramether you must specify if it is used as input +@param[in]+ as
+output +@param[out]+ or both +@param[inout]+. Paramethers and return value
+types must be of a type supported by +RsTypeSerializer+ which already support
+most basic types (+bool+, +std::string+...), +RsSerializable+ and containers of
+them like +std::vector<std::string>+. Paramethers passed by value and by
+reference of those types are both supported, while passing by pointer is not
+supported. If your paramether or return +class+/+struct+ type is not supported
+yet by +RsTypeSerializer+ most convenient approach is to make it derive from
++RsSerializable+ and implement +serial_process+ method like I did with
++RsGxsChannelGroup+.
+
+.Deriving RsGxsChannelGroup from RsSerializable in rsgxschannels.h
+[source,cpp]
+--------------------------------------------------------------------------------
+struct RsGxsChannelGroup : RsSerializable
+{
+	RsGroupMetaData mMeta;
+	std::string mDescription;
+	RsGxsImage mImage;
+
+	bool mAutoDownload;
+
+	/// @see RsSerializable
+	virtual void serial_process( RsGenericSerializer::SerializeJob j,
+	                             RsGenericSerializer::SerializeContext& ctx )
+	{
+		RS_SERIAL_PROCESS(mMeta);
+		RS_SERIAL_PROCESS(mDescription);
+		RS_SERIAL_PROCESS(mImage);
+		RS_SERIAL_PROCESS(mAutoDownload);
+	}
+};
+--------------------------------------------------------------------------------
+
+You can do the same recursively for any member of your +struct+ that is not yet
+supported by +RsTypeSerializer+.
+
+Some Retroshare {Cxx} API functions are asyncronous, historically RetroShare
+didn't follow a policy on how to expose asyncronous API so differents services
+and some times even differents method of the same service follow differents
+asyncronous patterns, thus making automatic generation of JSON API wrappers for
+those methods impractical. Instead of dealing with all those differents patterns
+I have chosed to support only one new pattern taking advantage of modern {Cxx}11
+and restbed features. On the {Cxx}11 side lambdas and +std::function+s are used,
+on the restbed side Server Side Events are used to send asyncronous results.
+
+Lets see an example so it will be much esier to understand.
+
+.RsGxsChannels::turtleSearchRequest asyncronous API
+[source,cpp]
+--------------------------------------------------------------------------------
+	/**
+	 * @brief Request remote channels search
+	 * @jsonapi{development}
+	 * @param[in] matchString string to look for in the search
+	 * @param multiCallback function that will be called each time a search
+	 * result is received
+	 * @param[in] maxWait maximum wait time in seconds for search results
+	 * @return false on error, true otherwise
+	 */
+	virtual bool turtleSearchRequest(
+	        const std::string& matchString,
+	        const std::function<void (const RsGxsGroupSummary& result)>& multiCallback,
+	        std::time_t maxWait = 300 ) = 0;
+--------------------------------------------------------------------------------
+
++RsGxsChannels::turtleSearchRequest(...)+ is an asyncronous method because it
+send a channel search request on turtle network and then everytime a result is
+received from the network +multiCallback+ is called and the result is passed as
+parameter. To be supported by the automatic JSON API wrappers generator an
+asyncronous method need a parameter of type +std::function<void (...)>+ called
++callback+ if the callback will be called only once or +multiCallback+ if the
+callback is expected to be called more then once like in this case.
+A second mandatory parameter is +maxWait+ of type +std::time_t+ it indicates the
+maximum amount of time in seconds for which the caller is willing to wait for
+results, in case the timeout is reached the callback will not be called anymore.
+
+[IMPORTANT]
+================================================================================
++callback+ and +multiCallback+ parameters documentation must *not* specify
++[in]+, +[out]+, +[inout]+, in Doxygen documentation as this would fool the
+automatic wrapper generator, and ultimately break the compilation.
+================================================================================
+
+.RsFiles::turtleSearchRequest asyncronous JSON API usage example
+[source,bash]
+--------------------------------------------------------------------------------
+$ cat turtle_search.json
+{
+    "matchString":"linux"
+}
+$ curl --data @turtle_search.json http://127.0.0.1:9092/rsFiles/turtleSearchRequest
+data: {"retval":true}
+
+data: {"results":[{"size":157631,"hash":"69709b4d01025584a8def5cd78ebbd1a3cf3fd05","name":"kill_bill_linux_1024x768.jpg"},{"size":192560,"hash":"000000000000000000009a93e5be8486c496f46c","name":"coffee_box_linux2.jpg"},{"size":455087,"hash":"9a93e5be8486c496f46c00000000000000000000","name":"Linux.png"},{"size":182004,"hash":"e8845280912ebf3779e400000000000000000000","name":"Linux_2_6.png"}]}
+
+data: {"results":[{"size":668,"hash":"e8845280912ebf3779e400000000000000000000","name":"linux.png"},{"size":70,"hash":"e8845280912ebf3779e400000000000000000000","name":"kali-linux-2016.2-amd64.txt.sha1sum"},{"size":3076767744,"hash":"e8845280912ebf3779e400000000000000000000","name":"kali-linux-2016.2-amd64.iso"},{"size":2780872,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux.bin"},{"size":917504,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux.lzma"},{"size":2278404096,"hash":"e8845280912ebf3779e400000000000000000000","name":"gentoo-linux-livedvd-amd64-multilib-20160704.iso"},{"size":151770333,"hash":"e8845280912ebf3779e400000000000000000000","name":"flashtool-0.9.23.0-linux.tar.7z"},{"size":2847372,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux.elf"},{"size":1310720,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux.gz"},{"size":987809,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux-lzma.elf"}]}
+
+--------------------------------------------------------------------------------
+
+By default JSON API methods requires client authentication and their wrappers
+are automatically generated by +json-api-generator+.
+In some cases methods need do be accessible without authentication such as
++rsLoginHelper/getLocations+ so in the doxygen documentaion they have the custom
+command +@jsonapi{RS_VERSION,unauthenticated}+.
+Other methods such as +/rsControl/rsGlobalShutDown+ need special care so they
+are marked with the custom doxygen command +@jsonapi{RS_VERSION,manualwrapper}+
+and their wrappers are not automatically generated but written manually into
++JsonApiServer::JsonApiServer(...)+.
+
+
+== A bit of history
+
+=== First writings about this
+
+The previous attempt of exposing a RetroShare JSON API is called +libresapi+ and
+unfortunatley it requires a bunch of boilerplate code when we want to expose
+something present in the {Cxx} API in the JSON API.
+
+As an example here you can see the libresapi that exposes part of the retroshare
+chat {Cxx} API and lot of boilerplate code just to convert {Cxx} objects to JSON
+
+https://github.com/RetroShare/RetroShare/blob/v0.6.4/libresapi/src/api/ChatHandler.cpp#L44
+
+To avoid the {Cxx} to JSON and back conversion boilerplate code I have worked out
+an extension to our {Cxx} serialization code so it is capable to serialize and
+deserialize to JSON you can see it in this pull request
+
+https://github.com/RetroShare/RetroShare/pull/1155
+
+So first step toward having a good API is to take advantage of the fact that RS
+is now capable of converting C++ objects from and to JSON.
+
+The current API is accessible via HTTP and unix socket, there is no
+authentication in both of them, so anyone having access to the HTTP server or to
+the unix socket can access the API without extra restrictions.
+Expecially for the HTTP API this is a big risk because also if the http server
+listen on 127.0.0.1 every application on the machine (even rogue javascript
+running on your web browser) can access that and for example on android it is
+not safe at all (because of that I implemented the unix socket access so at
+least in android API was reasonably safe) because of this.
+
+A second step to improve the API would be to implement some kind of API
+authentication mechanism (it would be nice that the mechanism is handled at API
+level and not at transport level so we can use it for any API trasport not just
+HTTP for example)
+
+The HTTP server used by libresapi is libmicrohttpd server that is very minimal,
+it doesn't provide HTTPS nor modern HTTP goodies, like server notifications,
+websockets etc. because the lack of support we have a token polling mechanism in
+libresapi to avoid polling for every thing but it is still ugly, so if we can
+completely get rid of polling in the API that would be really nice.
+I have done a crawl to look for a replacement and briefly looked at
+
+- https://www.gnu.org/software/libmicrohttpd/
+- http://wolkykim.github.io/libasyncd/
+- https://github.com/corvusoft/restbed
+- https://code.facebook.com/posts/1503205539947302/introducing-proxygen-facebook-s-c-http-framework/
+- https://github.com/cmouse/yahttp
+
+taking in account a few metrics like modern HTTP goodies support, license,
+platform support, external dependencies and documentation it seemed to me that
+restbed is the more appropriate.
+
+Another source of boilerplate code into libresapi is the mapping between JSON
+API requests and C++ API methods as an example you can look at this
+
+https://github.com/RetroShare/RetroShare/blob/v0.6.4/libresapi/src/api/ChatHandler.cpp#L158
+
+and this
+
+https://github.com/RetroShare/RetroShare/blob/v0.6.4/libresapi/src/api/ApiServer.cpp#L253
+
+The abstract logic of this thing is, when libreasapi get a request like
++/chat/initiate_distant_chat+ then call
++ChatHandler::handleInitiateDistantChatConnexion+ which in turn is just a
+wrapper of +RsMsgs::initiateDistantChatConnexion+ all this process is basically
+implemented as boilerplate code and would be unnecessary in a smarter design of
+the API because almost all the information needed is already present in the
+C++ API +libretroshare/src/retroshare+.
+
+So a third step to improve the JSON API would be to remove this source of
+boilerplate code by automatizing the mapping between C++ and JSON API call.
+
+This may result a little tricky as language parsing or other adevanced things
+may be required.
+
+Hope this dive is useful for you +
+Cheers +
+G10h4ck
+
+=== Second writings about this
+
+I have been investigating a bit more about:
+[verse, G10h4ck]
+________________________________________________________________________________
+So a third step to improve the JSON API would be to remove this source of
+boilerplate code by automatizing the mapping between C++ and JSON API call
+________________________________________________________________________________
+
+After spending some hours investigating this topic the most reasonable approach
+seems to:
+
+1. Properly document headers in +libretroshare/src/retroshare/+ in doxygen syntax
+specifying wihich params are input and/or output (doxygen sysntax for this is
++@param[in/out/inout]+) this will be the API documentation too.
+
+2. At compile time use doxygen to generate XML description of the headers and use
+the XML to generate the JSON api server stub.
+http://www.stack.nl/~dimitri/doxygen/manual/customize.html#xmlgenerator
+
+3. Enjoy

jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl

@@ -1 +0,0 @@
-../../libretroshare/src/jsonapi/async-method-wrapper-template.cpp.tmpl

jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl

@@ -0,0 +1,80 @@
+/*******************************************************************************
+ * RetroShare JSON API                                                         *
+ *                                                                             *
+ * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License version 3 as    *
+ * published by the Free Software Foundation.                                  *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+registerHandler( "$%apiPath%$",
+                 [this](const std::shared_ptr<rb::Session> session)
+{
+	const std::multimap<std::string, std::string> headers
+	{
+		{ "Connection", "keep-alive" },
+		{ "Content-Type", "text/event-stream" }
+	};
+	session->yield(rb::OK, headers);
+
+	size_t reqSize = session->get_request()->get_header("Content-Length", 0);
+	session->fetch( reqSize, [this](
+					const std::shared_ptr<rb::Session> session,
+					const rb::Bytes& body )
+	{
+		INITIALIZE_API_CALL_JSON_CONTEXT;
+
+		if( !checkRsServicePtrReady(
+		            $%instanceName%$, "$%instanceName%$", cAns, session ) )
+			return;
+
+$%paramsDeclaration%$
+
+$%inputParamsDeserialization%$
+
+		const std::weak_ptr<rb::Service> weakService(mService);
+		const std::weak_ptr<rb::Session> weakSession(session);
+		$%callbackName%$ = [weakService, weakSession]($%callbackParams%$)
+		{
+			auto session = weakSession.lock();
+			if(!session || session->is_closed()) return;
+
+			auto lService = weakService.lock();
+			if(!lService || lService->is_down()) return;
+
+$%callbackParamsSerialization%$
+
+			std::stringstream sStream;
+			sStream << "data: " << compactJSON << ctx.mJson << "\n\n";
+			const std::string message = sStream.str();
+
+			lService->schedule( [weakSession, message]()
+			{
+				auto session = weakSession.lock();
+				if(!session || session->is_closed()) return;
+				session->yield(message);
+				$%sessionEarlyClose%$
+			} );
+		};
+
+$%functionCall%$
+
+$%outputParamsSerialization%$
+
+		// return them to the API caller
+		std::stringstream message;
+		message << "data: " << compactJSON << cAns.mJson << "\n\n";
+		session->yield(message.str());
+		$%sessionDelayedClose%$
+	} );
+}, $%requiresAuth%$ );

jsonapi-generator/src/jsonapi-generator-doxygen.conf

@@ -1 +0,0 @@
-../../libretroshare/src/jsonapi/jsonapi-generator-doxygen.conf

jsonapi-generator/src/jsonapi-generator-doxygen.conf

@@ -0,0 +1,230 @@
+DOXYFILE_ENCODING      = UTF-8
+PROJECT_NAME           = "libretroshare"
+
+ALIASES += jsonapi{1}="\xmlonly<jsonapi minversion=\"\1\"/>\endxmlonly"
+ALIASES += jsonapi{2}="\xmlonly<jsonapi minversion=\"\1\" access=\"\2\"/>\endxmlonly"
+
+# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
+# documentation from any documented member that it re-implements.
+# The default value is: YES.
+
+INHERIT_DOCS           = YES
+
+# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
+# according to the Markdown format, which allows for more readable
+# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# The output of markdown processing is further processed by doxygen, so you can
+# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
+# case of backward compatibilities issues.
+# The default value is: YES.
+
+MARKDOWN_SUPPORT       = YES
+
+# When enabled doxygen tries to link words that correspond to documented
+# classes, or namespaces to their corresponding documentation. Such a link can
+# be prevented in individual cases by putting a % sign in front of the word or
+# globally by setting AUTOLINK_SUPPORT to NO.
+# The default value is: YES.
+
+AUTOLINK_SUPPORT       = YES
+
+
+# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in
+# documentation are documented, even if no documentation was available. Private
+# class members and static file members will be hidden unless the
+# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
+# Note: This will also disable the warnings about undocumented members that are
+# normally produced when WARNINGS is set to YES.
+# The default value is: NO.
+
+EXTRACT_ALL            = YES
+
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
+# undocumented members inside documented classes or files. If set to NO these
+# members will be included in the various overviews, but no documentation
+# section is generated. This option has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_MEMBERS     = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy. If set
+# to NO, these classes will be included in the various overviews. This option
+# has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_CLASSES     = NO
+
+
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+#INPUT                  =
+
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
+# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
+# documentation (see: https://www.gnu.org/software/libiconv/) for the list of
+# possible encodings.
+# The default value is: UTF-8.
+
+INPUT_ENCODING         = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
+# *.h) to filter out the source-files in the directories.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# read by doxygen.
+#
+# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
+# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
+# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
+# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08,
+# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf.
+
+FILE_PATTERNS          = *.c \
+                         *.cc \
+                         *.cxx \
+                         *.cpp \
+                         *.c++ \
+                         *.java \
+                         *.ii \
+                         *.ixx \
+                         *.ipp \
+                         *.i++ \
+                         *.inl \
+                         *.idl \
+                         *.ddl \
+                         *.odl \
+                         *.h \
+                         *.hh \
+                         *.hxx \
+                         *.hpp \
+                         *.h++ \
+                         *.cs \
+                         *.d \
+                         *.php \
+                         *.php4 \
+                         *.php5 \
+                         *.phtml \
+                         *.inc \
+                         *.m \
+                         *.markdown \
+                         *.md \
+                         *.mm \
+                         *.dox \
+                         *.py \
+                         *.pyw \
+                         *.f90 \
+                         *.f95 \
+                         *.f03 \
+                         *.f08 \
+                         *.f \
+                         *.for \
+                         *.tcl \
+                         *.vhd \
+                         *.vhdl \
+                         *.ucf \
+                         *.qsf
+
+# The RECURSIVE tag can be used to specify whether or not subdirectories should
+# be searched for input files as well.
+# The default value is: NO.
+
+RECURSIVE              = YES
+
+# The EXCLUDE tag can be used to specify files and/or directories that should be
+# excluded from the INPUT source files. This way you can easily exclude a
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+#
+# Note that relative paths are relative to the directory from which doxygen is
+# run.
+
+EXCLUDE                =
+
+# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
+# directories that are symbolic links (a Unix file system feature) are excluded
+# from the input.
+# The default value is: NO.
+
+EXCLUDE_SYMLINKS       = NO
+
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories.
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories for example use the pattern */test/*
+
+EXCLUDE_PATTERNS       =
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
+# AClass::ANamespace, ANamespace::*Test
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories use the pattern */test/*
+
+EXCLUDE_SYMBOLS        =
+
+
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output
+# The default value is: YES.
+
+GENERATE_HTML          = NO
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
+# The default value is: NO.
+
+GENERATE_XML           = YES
+
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all
+# C-preprocessor directives found in the sources and include files.
+# The default value is: YES.
+
+ENABLE_PREPROCESSING   = YES
+
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION        = NO
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF     = NO
+

jsonapi-generator/src/method-wrapper-template.cpp.tmpl

@@ -1 +0,0 @@
-../../libretroshare/src/jsonapi/method-wrapper-template.cpp.tmpl

jsonapi-generator/src/method-wrapper-template.cpp.tmpl

@@ -0,0 +1,50 @@
+/*******************************************************************************
+ * RetroShare JSON API                                                         *
+ *                                                                             *
+ * Copyright (C) 2018-2019  Gioacchino Mazzurco <gio@eigenlab.org>             *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+registerHandler( "$%apiPath%$",
+                 [](const std::shared_ptr<rb::Session> session)
+{
+	size_t reqSize = session->get_request()->get_header("Content-Length", 0);
+	session->fetch( reqSize, [](
+	                const std::shared_ptr<rb::Session> session,
+	                const rb::Bytes& body )
+	{
+		INITIALIZE_API_CALL_JSON_CONTEXT;
+
+		if( !checkRsServicePtrReady(
+		            $%instanceName%$, "$%instanceName%$", cAns, session ) )
+			return;
+
+$%paramsDeclaration%$
+
+		// deserialize input parameters from JSON
+$%inputParamsDeserialization%$
+
+		// call retroshare C++ API
+$%functionCall%$
+
+		// serialize out parameters and return value to JSON
+$%outputParamsSerialization%$
+
+		// return them to the API caller
+		DEFAULT_API_CALL_JSON_RETURN(rb::OK);
+	} );
+}, $%requiresAuth%$ );
+

libbitdht

@@ -1 +0,0 @@
-Subproject commit 2ddc86fb575a61170f4c06a00152e3e7dc74c8f4

libbitdht/src/README.txt

@@ -0,0 +1,25 @@
+# SPDX-FileCopyrightText: (C) 2004-2019 Retroshare Team <contact@retroshare.cc>
+# SPDX-License-Identifier: CC0-1.0
+
+What's in the Package
+---------------------------------------------------------------
+bitdht          - base BitDHT Code.
+util		- generic utils for networking and threading.
+udp		- UDP interfacing code.
+lib             - Where the library is created.
+tests           - basic unit tests.
+example		- example code of how to use libbitdht.
+libbitdht.pro	- build script for Qt's qmake.
+README.txt	- this file.
+
+
+HOWTO libbitdht.
+----------------------------------------------
+This version is build using Qt's qmake system.
+
+1) Install Qt's qmake system:  libqt-dev
+2) type ./qmake
+3) type ./make
+4) check out the example and tests to learn how to interface with libbitdht.
+
+

libbitdht/src/bitdht/bdaccount.cc

@@ -0,0 +1,133 @@
+/*******************************************************************************
+ * bitdht/bdaccount.cc                                                         *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2011 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "bitdht/bdaccount.h"
+
+#include <string.h>
+#include <stdlib.h>
+
+#include <iostream>
+#include <iomanip>
+
+
+#define LPF_FACTOR  (0.90)
+
+bdAccount::bdAccount()
+	:mNoStats(BDACCOUNT_NUM_ENTRIES), 
+	mCountersOut(BDACCOUNT_NUM_ENTRIES), mCountersRecv(BDACCOUNT_NUM_ENTRIES), 
+	mLpfOut(BDACCOUNT_NUM_ENTRIES), mLpfRecv(BDACCOUNT_NUM_ENTRIES), 
+	mLabel(BDACCOUNT_NUM_ENTRIES)
+{
+
+	mLabel[BDACCOUNT_MSG_OUTOFDATEPING] 	=	 "OUTOFDATEPING  ";
+	mLabel[BDACCOUNT_MSG_PING] 		=	 "PING           ";
+	mLabel[BDACCOUNT_MSG_PONG] 		=	 "PONG           ";
+	mLabel[BDACCOUNT_MSG_QUERYNODE] 	=	 "QUERYNODE      ";
+	mLabel[BDACCOUNT_MSG_QUERYHASH] 	=	 "QUERYHASH      ";
+	mLabel[BDACCOUNT_MSG_REPLYFINDNODE] 	=	 "REPLYFINDNODE  ";
+	mLabel[BDACCOUNT_MSG_REPLYQUERYHASH] 	=	 "REPLYQUERYHASH ";
+
+	mLabel[BDACCOUNT_MSG_POSTHASH] 		=	 "POSTHASH       ";
+	mLabel[BDACCOUNT_MSG_REPLYPOSTHASH] 	=	 "REPLYPOSTHASH  ";
+
+	mLabel[BDACCOUNT_MSG_CONNECTREQUEST] 	=	 "CONNECTREQUEST ";
+	mLabel[BDACCOUNT_MSG_CONNECTREPLY] 	=	 "CONNECTREPLY   ";
+	mLabel[BDACCOUNT_MSG_CONNECTSTART] 	=	 "CONNECTSTART   ";
+	mLabel[BDACCOUNT_MSG_CONNECTACK] 	=	 "CONNECTACK     ";
+	
+	resetStats();
+}
+
+
+void bdAccount::incCounter(uint32_t idx, bool out)
+{
+	if ((signed) idx > mNoStats-1)
+	{
+		std::cerr << "bdAccount::incCounter() Invalid Index";
+		std::cerr << std::endl;
+	}
+
+	if (out)
+	{
+		mCountersOut[idx]++;
+	}
+	else
+	{
+		mCountersRecv[idx]++;
+	}
+	return;
+}
+
+
+void bdAccount::doStats()
+{
+	int i;
+	for(i = 0; i < mNoStats; i++)
+	{
+		mLpfOut[i] *= (LPF_FACTOR) ;
+		mLpfOut[i] += (1.0 - LPF_FACTOR) * mCountersOut[i];
+
+		mLpfRecv[i] *= (LPF_FACTOR) ;
+		mLpfRecv[i] += (1.0 - LPF_FACTOR) * mCountersRecv[i];
+	}
+	resetCounters();
+}
+
+void bdAccount::printStats(std::ostream &out)
+{
+	int i;
+	out << "  Send                                                 Recv: ";
+	out << std::endl;
+	for(i = 0; i < mNoStats; i++)
+	{
+
+		out << "Send" << mLabel[i] << " : " << std::setw(10) << mLpfOut[i];
+		out << "          ";
+		out << "Recv" << mLabel[i] << " : " << std::setw(10) << mLpfRecv[i];
+		out << std::endl;
+	}
+}
+
+void bdAccount::resetCounters()
+{
+	int i;
+	for(i = 0; i < mNoStats; i++)
+	{
+		mCountersOut[i] = 0;
+		mCountersRecv[i] = 0;
+	}
+}
+
+void bdAccount::resetStats()
+{
+	int i;
+	for(i = 0; i < mNoStats; i++)
+	{
+		mLpfOut[i] = 0;
+		mLpfRecv[i] = 0;
+	}
+	resetCounters();
+}
+
+
+
+

libbitdht/src/bitdht/bdaccount.h

@@ -1,7 +1,12 @@
+#ifndef BITDHT_ACCOUNT_H
+#define BITDHT_ACCOUNT_H
+
 /*******************************************************************************
- * gui/NetworkView.h                                                           *
+ * bitdht/bdaccount.h                                                          *
  *                                                                             *
- * Copyright (c) 2008 Robert Fernie    <retroshare.project@gmail.com>          *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2011 by Robert Fernie <bitdht@lunamutt.com>                       *
  *                                                                             *
  * This program is free software: you can redistribute it and/or modify        *
  * it under the terms of the GNU Affero General Public License as              *
@@ -18,48 +23,54 @@
  *                                                                             *
  *******************************************************************************/
 
-#pragma once
+#include <vector>
+#include <string>
+#include <inttypes.h>
 
-#include <QGraphicsScene>
+#define 	BDACCOUNT_MSG_OUTOFDATEPING		0
+#define 	BDACCOUNT_MSG_PING			1
+#define 	BDACCOUNT_MSG_PONG			2
+#define 	BDACCOUNT_MSG_QUERYNODE			3
+#define 	BDACCOUNT_MSG_QUERYHASH			4
+#define 	BDACCOUNT_MSG_REPLYFINDNODE		5
+#define 	BDACCOUNT_MSG_REPLYQUERYHASH		6
 
-#include "retroshare/rsevents.h"
+#define 	BDACCOUNT_MSG_POSTHASH			7
+#define 	BDACCOUNT_MSG_REPLYPOSTHASH		8
 
-#include "retroshare-gui/mainpage.h"
-#include "ui_FriendServerControl.h"
+#define 	BDACCOUNT_MSG_CONNECTREQUEST		9
+#define 	BDACCOUNT_MSG_CONNECTREPLY		10
+#define 	BDACCOUNT_MSG_CONNECTSTART		11
+#define 	BDACCOUNT_MSG_CONNECTACK		12
 
-class FriendServerControl : public MainPage, public Ui::FriendServerControl
+#define 	BDACCOUNT_NUM_ENTRIES 			13
+
+class bdAccount
 {
-    Q_OBJECT
+	public:
 
-public:
-    FriendServerControl(QWidget *parent = 0);
-    virtual ~FriendServerControl();
+	bdAccount();
 
-protected slots:
-    void onOnOffClick(bool b);
-    void onOnionAddressEdit(const QString &);
-    void onOnionPortEdit(int);
-    void onNbFriendsToRequestsChanged(int n);
-    void checkServerAddress();
-    void onAutoAddFriends(bool b);
-    void launchStatusContextMenu(QPoint p);
-    void makeFriend();
+	void incCounter(uint32_t idx, bool out);
+	void doStats();
+	void printStats(std::ostream &out);	
+	void resetCounters();
+	void resetStats();
 
-private:
-    RsPeerId getCurrentPeer();
-    void updateContactsStatus();
-    void updateFriendServerStatusIcon(bool ok);
-    void handleEvent_main_thread(std::shared_ptr<const RsEvent> event);
+	private:
 
-    QTimer *mConnectionCheckTimer;
-    QMovie *mCheckingServerMovie;
-    bool 	mCurrentlyCheckingServerAddress;
+	int mNoStats;
 
-    RsEventsHandlerId_t mEventHandlerId_fs;
-    RsEventsHandlerId_t mEventHandlerId_peer;
+	std::vector<double> mCountersOut;
+	std::vector<double> mCountersRecv;
 
-    QAction *makeFriend_ACT;
-    QAction *unmakeFriend_ACT;
-    QAction *removePeer_ACT;
-    QAction *removePeerPermanently_ACT;
+	std::vector<double> mLpfOut;
+	std::vector<double> mLpfRecv;
+
+	std::vector<std::string> mLabel;
+	// Statistics.
 };
+
+
+
+#endif // BITDHT_ACCOUNT_H

libbitdht/src/bitdht/bdboot_generate.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+<<LICENSE
+
+Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+LICENSE
+
+<<README
+Generate a clean DHT bootstrap node list.
+Feeds on previously known nodes from standard input and a few well known
+mainline DHT bootstrap nodes. Prints only nodes that appears to be active to a
+quick nmap check. Make sure your internet connection is working well before
+using this.
+
+Example usage:
+--------------------------------
+cat bdboot.txt | bdboot_generate.sh | tee /tmp/bdboot_generated.txt
+cat /tmp/bdboot_generated.txt | sort -u > bdboot.txt
+--------------------------------
+README
+
+function check_dht_host()
+{
+	mHost="$1"
+	mPort="$2"
+	
+	sudo nmap -oG - -sU -p $mPort $mHost | grep open | \
+		awk '{print $2" "$5}' | awk -F/ '{print $1}'
+}
+
+cat | while read line; do
+	hostIP="$(echo $line | awk '{print $1}')"
+	hostPort="$(echo $line | awk '{print $2}')"
+	check_dht_host $hostIP $hostPort
+done
+
+check_dht_host router.utorrent.com 6881
+check_dht_host router.bittorrent.com 6881
+check_dht_host dht.libtorrent.org 25401
+check_dht_host dht.transmissionbt.com 6881
+

libbitdht/src/bitdht/bdconnection.h

@@ -0,0 +1,314 @@
+/*******************************************************************************
+ * bitdht/bdconnection.h                                                       *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2011 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#ifndef BITDHT_CONNECTION_H
+#define BITDHT_CONNECTION_H
+
+#include "bitdht/bdiface.h"
+
+class bdQueryManager;
+class bdNodePublisher;
+
+/************************************************************************************************************
+************************************** ProxyTuple + Connection State ****************************************
+************************************************************************************************************/
+
+#define BITDHT_CONNREQUEST_READY		1
+#define BITDHT_CONNREQUEST_PAUSED		2
+#define BITDHT_CONNREQUEST_INPROGRESS		3
+#define BITDHT_CONNREQUEST_EXTCONNECT		4
+#define BITDHT_CONNREQUEST_DONE			5
+
+#define BITDHT_CONNREQUEST_TIMEOUT_CONNECT	300  // MAKE THIS LARGE - SHOULD NEVER HAPPEN.
+#define BITDHT_CONNREQUEST_TIMEOUT_INPROGRESS	30
+#define BITDHT_CONNREQUEST_MAX_AGE		60
+
+
+#define BITDHT_CONNECTION_WAITING_AUTH		1
+#define BITDHT_CONNECTION_WAITING_REPLY		2
+#define BITDHT_CONNECTION_WAITING_START		3
+#define BITDHT_CONNECTION_WAITING_ACK		4
+#define BITDHT_CONNECTION_COMPLETED		5
+
+
+#define BD_CONNECTION_START_RETRY_PERIOD	3  // Should only take a couple of seconds to get reply.
+#define BD_CONNECTION_START_MAX_RETRY		3
+#define BD_CONNECTION_MAX_TIMEOUT		20 /* should be quick */
+
+
+
+class bdProxyTuple 
+{
+	public:
+	bdProxyTuple() { return; }
+	bdProxyTuple(bdNodeId *s, bdNodeId *p, bdNodeId *d)
+	:srcId(*s), proxyId(*p), destId(*d) { return; }
+
+	bdNodeId srcId;
+	bdNodeId proxyId;
+	bdNodeId destId;
+};
+
+std::ostream &operator<<(std::ostream &out, const bdProxyTuple &t);
+int operator<(const bdProxyTuple &a, const bdProxyTuple &b);
+int operator==(const bdProxyTuple &a, const bdProxyTuple &b);
+
+
+class bdConnection
+{
+	public:
+	bdConnection();
+
+	/** Functions to tweak the connection status */
+
+	// User initialised Connection.
+	int ConnectionSetup(bdId *proxyId, bdId *srcConnAddr, bdId *destConnAddr, int mode, int delay);
+	int ConnectionSetupDirect(bdId *destId, bdId *srcConnAddr);
+
+	// Initialise a new Connection. (receiving a Connection Request)
+	int ConnectionRequestDirect(bdId *id, bdId *srcConnAddr, bdId *destConnAddr);
+	int ConnectionRequestProxy(bdId *id, bdId *srcConnAddr, bdNodeId *ownId, bdId *destConnAddr, int mode, int delay);
+	int ConnectionRequestEnd(bdId *id, bdId *srcConnAddr, bdId *destConnAddr, int mode);
+
+	// Setup Finishing Stage, (receiving a Connection Reply).
+	int upgradeProxyConnectionToFinish(bdId *id, bdId *srcConnAddr, bdId *destConnAddr, int mode, int delay, int status);
+
+	int AuthoriseDirectConnection(bdId *srcId, bdId *proxyId, bdId *destId, int mode, int loc);
+	int AuthoriseProxyConnection(bdId *srcId, bdId *proxyId, bdId *destId, int mode, int loc, int bandwidth);
+	int AuthoriseEndConnection(bdId *srcId, bdId *proxyId, bdId *destId, int mode, int loc, int delay);
+
+	int CompleteConnection(bdId *id, bdId *srcConnAddr, bdId *destConnAddr, int bandwidth, int delay);
+
+	int checkForDefaultConnectAddress();
+
+	/* Connection State, and TimeStamp of Update */
+	int mState;
+	time_t mLastEvent;
+
+	/* Addresses of Start/Proxy/End Nodes */
+	bdId mSrcId;
+	bdId mDestId;
+	bdId mProxyId;
+
+	/* Where we are in the connection, 
+	 * and what connection mode.
+	 */
+	int mPoint;
+	int mMode;
+
+	/* must have ip:ports of connection ends (if proxied) */
+	bdId mSrcConnAddr;
+	bdId mDestConnAddr;
+
+	int mBandwidth;
+	int mMaxDelay;
+	time_t mConnectionStartTS;
+
+	/* START/ACK Finishing ****/
+	time_t mLastStart;   /* timer for retries */
+	int mRetryCount;     /* retry counter */
+
+	bool mSrcAck;
+	bool mDestAck;
+
+	// Completion TS.
+	time_t mCompletedTS;
+
+
+};
+
+#define BD_PI_SRC_UNKNOWN					0
+#define BD_PI_SRC_QUERYRESULT				1
+#define BD_PI_SRC_QUERYPROXY				2
+#define BD_PI_SRC_NODESPACE_FRIEND			3
+#define BD_PI_SRC_NODESPACE_SERVER			4
+#define BD_PI_SRC_NODESPACE_ENGINEVERSION	5
+#define BD_PI_SRC_ADDGOODPROXY				6
+
+
+class bdProxyId
+{
+public:
+	bdProxyId(const bdId &in_id, uint32_t in_srctype, uint32_t in_errcode)
+	:id(in_id), srcType(in_srctype), errcode(in_errcode) { return; }
+
+	bdProxyId() :srcType(BD_PI_SRC_UNKNOWN), errcode(0) { return; }
+
+	std::string proxySrcType() const;
+
+	bdId id;
+	uint32_t srcType;
+	uint32_t errcode;
+};
+
+
+class bdConnectionRequest
+{
+public:
+	bdConnectionRequest() : mMode(0), mState(0), mStateTS(0), mPauseTS(0), mErrCode(0), mDelay(0), mRequestTS(0), mRecycled(0), mCurrentSrcType(0)
+	{
+		bdsockaddr_clear(&mLocalAddr);
+	}
+
+public:
+	int setupDirectConnection(struct sockaddr_in *laddr, bdNodeId *target);
+	int setupProxyConnection(struct sockaddr_in *laddr, bdNodeId *target, uint32_t mode, uint32_t delay);
+
+	int addGoodProxy(const bdId *srcId);
+	int checkGoodProxyPeer(const bdId *Id);
+
+	bdNodeId mTarget;
+	struct sockaddr_in mLocalAddr;
+	int mMode;
+
+	int mState;
+	time_t mStateTS;
+
+	time_t mPauseTS;
+	uint32_t mErrCode;
+	
+	int mDelay;
+	time_t mRequestTS;   // reference Time for mDelay.
+
+	std::list<bdProxyId> mGoodProxies;
+	std::list<bdId> mPotentialProxies;
+	//std::list<bdId> mGoodProxies;
+	int mRecycled;
+
+	bdId mCurrentAttempt;
+	uint32_t mCurrentSrcType;
+
+	std::list<bdProxyId> mPeersTried;
+	//std::list<bdId> mPeersTried;
+};
+
+std::ostream &operator<<(std::ostream &out, const bdConnectionRequest &req);
+std::ostream &operator<<(std::ostream &out, const bdConnection &conn);
+
+
+/*********
+ * The Connection Management Class.
+ * this encapsulates all of the functionality..
+ * except for a couple of message in/outs + callback.
+ */
+
+class bdConnectManager
+{
+	public:
+
+	bdConnectManager(bdNodeId *ownid, bdSpace *space, bdQueryManager *qmgr, bdDhtFunctions *fns, bdNodePublisher *pub);	
+
+
+	/* connection functions */
+	void requestConnection(bdNodeId *id, uint32_t modes);
+	void allowConnection(bdNodeId *id, uint32_t modes);
+
+
+	/* high level */
+
+        void shutdownConnections();
+        void printConnections();
+
+	/* Connections: Configuration */
+	void defaultConnectionOptions();
+	virtual void setConnectionOptions(uint32_t allowedModes, uint32_t flags);
+
+	/* Connections: Initiation */
+
+	int requestConnection(struct sockaddr_in *laddr, bdNodeId *target, uint32_t mode, uint32_t delay, uint32_t start);
+	int requestConnection_direct(struct sockaddr_in *laddr, bdNodeId *target);	
+	int requestConnection_proxy(struct sockaddr_in *laddr, bdNodeId *target, uint32_t mode, uint32_t delay);
+
+	int killConnectionRequest(struct sockaddr_in *laddr, bdNodeId *target, uint32_t mode);
+
+	int checkExistingConnectionAttempt(bdNodeId *target);
+	void addPotentialConnectionProxy(const bdId *srcId, const bdId *target);
+	void updatePotentialConnectionProxy(const bdId *id, uint32_t mode);
+
+	int checkPeerForFlag(const bdId *id, uint32_t with_flag);
+
+	int tickConnections();
+	void iterateConnectionRequests();
+	int startConnectionAttempt(bdConnectionRequest *req);
+
+	// internal Callback -> normally continues to callbackConnect().
+	void callbackConnectRequest(bdId *srcId, bdId *proxyId, bdId *destId, 
+				int mode, int point, int param, int cbtype, int errcode);
+
+	/* Connections: Outgoing */
+
+	int  startConnectionAttempt(bdId *proxyId, bdId *srcConnAddr, bdId *destConnAddr, int mode, int delay);
+	void AuthConnectionOk(bdId *srcId, bdId *proxyId, bdId *destId, int mode, int loc, int bandwidth, int delay);
+	void AuthConnectionNo(bdId *srcId, bdId *proxyId, bdId *destId, int mode, int loc, int errcode);
+	void iterateConnections();
+
+
+	/* Connections: Utility State */
+
+	bdConnection *findExistingConnection(bdNodeId *srcId, bdNodeId *proxyId, bdNodeId *destId);
+	bdConnection *newConnection(bdNodeId *srcId, bdNodeId *proxyId, bdNodeId *destId);
+	int cleanConnection(bdNodeId *srcId, bdNodeId *proxyId, bdNodeId *destId);
+
+	int determinePosition(bdNodeId *sender, bdNodeId *src, bdNodeId *dest);
+	int determineProxyId(bdNodeId *sender, bdNodeId *src, bdNodeId *dest, bdNodeId *proxyId);
+
+	bdConnection *findSimilarConnection(bdNodeId *srcId, bdNodeId *destId);
+	bdConnection *findExistingConnectionBySender(bdId *sender, bdId *src, bdId *dest);
+	bdConnection *newConnectionBySender(bdId *sender, bdId *src, bdId *dest);
+	int cleanConnectionBySender(bdId *sender, bdId *src, bdId *dest);
+
+	// Overloaded Generalised Connection Callback.
+	virtual void callbackConnect(bdId *srcId, bdId *proxyId, bdId *destId, 
+				int mode, int point, int param, int cbtype, int errcode);
+
+	/* Connections: */
+	int recvedConnectionRequest(bdId *id, bdId *srcConnAddr, bdId *destConnAddr, int mode, int delay);
+	int recvedConnectionReply(bdId *id, bdId *srcConnAddr, bdId *destConnAddr, int mode, int delay, int status);
+	int recvedConnectionStart(bdId *id, bdId *srcConnAddr, bdId *destConnAddr, int mode, int delayOrBandwidth);
+	int recvedConnectionAck(bdId *id, bdId *srcConnAddr, bdId *destConnAddr, int mode);
+
+	/* setup Relay Mode */
+	void setRelayMode(uint32_t mode);
+
+	private:
+
+	std::map<bdProxyTuple, bdConnection> mConnections;
+	std::map<bdNodeId, bdConnectionRequest> mConnectionRequests;
+
+        uint32_t mConfigAllowedModes;
+        bool mConfigAutoProxy;
+
+	uint32_t mRelayMode;
+
+	/****************************** Connection Code (in bdconnection.cc) ****************************/
+
+	private:
+
+	bdNodeId mOwnId;
+	bdSpace *mNodeSpace;
+	bdQueryManager *mQueryMgr;
+	bdDhtFunctions *mFns;
+	bdNodePublisher *mPub;
+};
+
+
+#endif // BITDHT_CONNECTION_H

libbitdht/src/bitdht/bdfilter.cc

@@ -0,0 +1,318 @@
+/*******************************************************************************
+ * bitdht/bdfilter.cc                                                          *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "bitdht/bdfilter.h"
+#include "bitdht/bdmanager.h"
+#include "util/bdfile.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <iostream>
+#include <time.h>
+
+/**
+ * #define DEBUG_FILTER 1
+**/
+#define BDFILTER_ENTRY_DROP_PERIOD	(7 * 24 * 3600)
+
+bdFilter::bdFilter(const std::string &fname, const bdNodeId *ownid,  uint32_t filterFlags, bdDhtFunctions *fns, bdNodeManager *manager)
+{
+	/* */
+    mOwnId = *ownid;
+    mFns = fns;
+    mFilename = fname ;
+
+    loadBannedIpFile() ;
+
+    mFilterFlags = filterFlags;
+    mNodeManager = manager;
+}
+
+void bdFilter::writeBannedIpFile()
+{
+    std::string filetmp = mFilename + ".tmp" ;
+
+    FILE *fd = fopen(filetmp.c_str(), "w");
+
+    if (!fd)
+    {
+        std::cerr << "(EE) bdFilter::writeBannedIpFile() FAILED to Open File " << mFilename << std::endl;
+        return;
+    }
+
+    for( std::map<uint32_t,bdFilteredPeer>::iterator it=mFiltered.begin();it!=mFiltered.end();++it)
+    {
+        fprintf(fd, "%s %u %lu %lu\n", bdnet_inet_ntoa(it->second.mAddr.sin_addr).c_str(), it->second.mFilterFlags, it->second.mFilterTS, it->second.mLastSeen) ;
+#ifdef DEBUG_FILTER
+        fprintf(stderr, "Storing Peer Address: %s \n", bdnet_inet_ntoa(it->second.mAddr.sin_addr).c_str()) ;
+#endif
+
+    }
+    fclose(fd);
+
+    if(!bdFile::renameFile(filetmp,mFilename))
+        std::cerr << "Could not rename file !!" << std::endl;
+#ifdef DEBUG_FILTER
+    else
+        std::cerr << "Successfully renamed file " << filetmp << " to " << mFilename << std::endl;
+#endif
+}
+
+void bdFilter::loadBannedIpFile()
+{
+        char line[10240];
+        char addr_str[10240];
+
+        struct sockaddr_in addr;
+        memset(&addr, 0, sizeof(struct sockaddr_in));
+        addr.sin_family = PF_INET;
+
+    FILE *fd = fopen(mFilename.c_str(),"r") ;
+
+    if(fd == NULL)
+    {
+        std::cerr << "(EE) Cannot load filter file " << mFilename << std::endl;
+        return ;
+    }
+
+        while(line == fgets(line, 10240, fd))
+        {
+        uint32_t filter_flags ;
+        unsigned long long int filter_ts ;
+        unsigned long long int last_seen ;
+
+            if (4 == sscanf(line, "%s %u %llu %llu", addr_str, &filter_flags,&filter_ts,&last_seen))
+            {
+                if (bdnet_inet_aton(addr_str, &(addr.sin_addr)))
+                {
+                    addr.sin_port = 0;
+
+                    bdFilteredPeer peer;
+
+                    peer.mAddr = addr;
+                    peer.mFilterTS = filter_ts;
+                    peer.mLastSeen = last_seen;
+                    peer.mFilterFlags = filter_flags;
+
+            mFiltered[addr.sin_addr.s_addr] = peer ;
+    #ifdef DEBUG_FILTER
+                    std::cerr << "Loaded filtered IP: " << std::string(addr_str) << " last seen: " << last_seen << ", TS=" << filter_ts << std::endl;
+    #endif
+                }
+            }
+        }
+
+        fclose(fd);
+}
+
+//bool bdFilter::filtered(std::list<bdFilteredPeer> &answer)
+//{
+//	answer = mFiltered;
+//	return (answer.size() > 0);
+//}
+
+bool bdFilter::filteredIPs(std::list<struct sockaddr_in> &answer)
+{
+    std::map<uint32_t,bdFilteredPeer>::iterator it;
+	for(it = mFiltered.begin(); it != mFiltered.end(); it++)
+	{
+        answer.push_back(it->second.mAddr);
+	}
+	return (answer.size() > 0);
+}
+
+int bdFilter::checkPeer(const bdId *id, uint32_t mode)
+{
+	bool add = false;
+	uint32_t flags = 0;
+	if ((mFilterFlags & BITDHT_FILTER_REASON_OWNID) && 
+			isOwnIdWithoutBitDhtFlags(id, mode))
+	{
+		add = true;
+		flags |= BITDHT_FILTER_REASON_OWNID;
+	}
+
+	if (add)
+	{
+        bool isNew = addPeerToFilter(id->addr, flags);
+
+		if (isNew)
+		{
+			return 1;
+		}
+	}
+
+	return 0;
+}
+
+int bdFilter::addPeerToFilter(const struct sockaddr_in& addr, uint32_t flags)
+{
+    std::map<uint32_t,bdFilteredPeer>::iterator it = mFiltered.find(addr.sin_addr.s_addr) ;
+
+    if(it != mFiltered.end())
+	{
+            it->second.mLastSeen = time(NULL);
+            it->second.mFilterFlags |= flags;
+    }
+    else
+    {
+        time_t now = time(NULL);
+        bdFilteredPeer fp;
+
+        fp.mAddr = addr;
+        fp.mAddr.sin_port = 0;
+        fp.mFilterFlags = flags;
+        fp.mFilterTS = now;
+        fp.mLastSeen = now;
+
+        uint32_t saddr = addr.sin_addr.s_addr;
+
+        mFiltered[saddr] = fp;
+
+        std::cerr << "Adding New Banned Ip Address: " << bdnet_inet_ntoa(addr.sin_addr);
+        std::cerr << std::endl;
+    }
+    writeBannedIpFile() ;
+
+    return true;
+}
+
+// void bdFilter::loadFilteredPeers(const std::list<bdFilteredPeer>& peers)
+// {
+//     for(std::list<bdFilteredPeer>::iterator it = peers.begin(); it != peers.end();++it)
+//     {
+// #ifdef DEBUG_FILTER
+//         std::cerr << "Loading filtered peer " << inet_ntoa(it->mAddr.sin_addr) << " Flags: " << it->mFilterFlags << " FilterTS: "
+//                   << now - it->mFilterTS << " LastSeen: " << now - it->mLastSeen << std::endl;
+// #endif
+//         uint32_t saddr = it->mAddr.sin_addr.s_addr;
+//         mFiltered[saddr] = *it ;
+//     }
+// }
+void bdFilter::getFilteredPeers(std::list<bdFilteredPeer>& peers)
+{
+    for(std::map<uint32_t,bdFilteredPeer>::iterator it = mFiltered.begin(); it != mFiltered.end();++it)
+        peers.push_back(it->second) ;
+}
+/* fast check if the addr is in the structure */
+int bdFilter::addrOkay(struct sockaddr_in *addr)
+{
+	// first check upper layer
+	bool isAvailable, isBanned;
+	mNodeManager->doIsBannedCallback(addr, &isAvailable, &isBanned);
+
+	if(isAvailable) {
+#ifdef DEBUG_FILTER
+		std::cerr << "bdFilter::addrOkay addr: " << inet_ntoa(addr->sin_addr) << " result from upper layer: " << (isBanned ? "banned" : "ok") << std::endl;
+#endif
+		return !isBanned;
+	} else {
+		// fallback to own ban list
+
+		std::map<uint32_t,bdFilteredPeer>::const_iterator it = mFiltered.find(addr->sin_addr.s_addr);
+		if (it == mFiltered.end())
+			return 1; // Address is Okay
+	}
+
+#ifdef DEBUG_FILTER
+	std::cerr << "Detected Packet From Banned Ip Address: " << inet_ntoa(addr->sin_addr);
+	std::cerr << std::endl;
+#endif
+	return 0;
+}
+
+
+bool bdFilter::isOwnIdWithoutBitDhtFlags(const bdId *id, uint32_t peerFlags)
+{
+	if (peerFlags & BITDHT_PEER_STATUS_RECV_PONG)
+	{
+		if (peerFlags & BITDHT_PEER_STATUS_DHT_ENGINE)
+		{
+			/* okay! */
+			return false;
+		}
+
+		/* now check distance */
+		bdMetric dist;
+		mFns->bdDistance(&mOwnId, &(id->id), &dist);
+		int bucket = mFns->bdBucketDistance(&dist);
+
+		/* if they match us... kill it */
+		if (bucket == 0)
+		{
+			return true;
+		}
+	}
+	return false;
+}
+
+
+/* periodically we want to cleanup the filter....
+ * if we haven't had an IP address reported as filtered for several hours.
+ * remove it from the list.
+ */
+
+bool bdFilter::cleanupFilter()
+{
+#ifdef DEBUG_FILTER
+    std::cerr << "bdFilter: Checking current filter List:" << std::endl;
+#endif
+
+	time_t now = time(NULL);
+	time_t dropTime = now - BDFILTER_ENTRY_DROP_PERIOD;
+
+    for(std::map<uint32_t,bdFilteredPeer>::iterator it = mFiltered.begin(); it != mFiltered.end();)
+    {
+#ifdef DEBUG_FILTER
+        std::cerr << "\t" << bdnet_inet_ntoa(it->second.mAddr.sin_addr);
+        std::cerr << " Flags: " << it->second.mFilterFlags;
+        std::cerr << " FilterTS: " << now - it->second.mFilterTS;
+        std::cerr << " LastSeen: " << now - it->second.mLastSeen;
+#endif
+
+        if (it->second.mLastSeen < dropTime)
+        {
+            /* remove from filter */
+#ifdef DEBUG_FILTER
+            std::cerr << " OLD DROPPING" << std::endl;
+#endif
+            std::map<uint32_t,bdFilteredPeer>::iterator tmp(it) ;
+            ++tmp ;
+
+            mFiltered.erase(it);
+            it = tmp ;
+        }
+        else
+        {
+#ifdef DEBUG_FILTER
+            std::cerr << " OK" << std::endl;
+#endif
+            it++;
+        }
+    }
+
+	return true;
+}
+
+
+

libbitdht/src/bitdht/bdfilter.h

@@ -0,0 +1,84 @@
+/*******************************************************************************
+ * bitdht/bdfilter.h                                                           *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#ifndef BITDHT_FILTER_H
+#define BITDHT_FILTER_H
+
+#include "bitdht/bdiface.h"
+#include <set>
+
+/* Query result flags are in bdiface.h */
+
+#define BITDHT_FILTER_REASON_OWNID	0x0001
+
+class bdFilteredPeer
+{
+    public:
+    struct sockaddr_in mAddr;
+    uint32_t mFilterFlags; /* reasons why we are filtering */
+    time_t mFilterTS;
+    time_t mLastSeen;
+};
+
+class bdNodeManager;
+
+class bdFilter
+{
+	public:
+    bdFilter(const std::string& fname, const bdNodeId *ownid, uint32_t filterFlags, bdDhtFunctions *fns, bdNodeManager *manager);
+
+	// get the answer.
+//bool	filtered(std::list<bdFilteredPeer> &answer);
+bool	filteredIPs(std::list<struct sockaddr_in> &answer);
+
+//void    loadFilteredPeers(const std::list<bdFilteredPeer>& peers) ;
+void 	getFilteredPeers(std::list<bdFilteredPeer> &peers);
+
+int 	checkPeer(const bdId *id, uint32_t peerFlags);
+
+int 	addrOkay(struct sockaddr_in *addr);
+int 	addPeerToFilter(const struct sockaddr_in &addr, uint32_t flags);
+
+bool 	cleanupFilter();
+
+void loadBannedIpFile() ;
+void writeBannedIpFile() ;
+
+private:
+
+bool	isOwnIdWithoutBitDhtFlags(const bdId *id, uint32_t peerFlags);
+
+	// searching for
+	bdNodeId mOwnId;
+	uint32_t mFilterFlags;
+
+	std::map<uint32_t,bdFilteredPeer> mFiltered;
+	bdDhtFunctions *mFns;
+	std::string mFilename ;
+
+	// have access to the manager for isBanned callback
+	bdNodeManager* mNodeManager;
+};
+
+
+#endif
+

libbitdht/src/bitdht/bdfriendlist.cc

@@ -0,0 +1,243 @@
+/*******************************************************************************
+ * bitdht/bdfriendlist.cc                                                      *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "bitdht/bdfriendlist.h"
+#include "bitdht/bdstddht.h"
+#include "bitdht/bdpeer.h"
+
+#include <iostream>
+
+/*****
+ * #define DEBUG_FRIENDLIST	1
+ ****/
+
+bdFriendEntry::bdFriendEntry()
+{
+	mFlags = 0;
+	mLastSeen = 0;
+}
+
+bool bdFriendEntry::addrKnown(struct sockaddr_in *addr)
+{
+	if (mFlags & BD_FRIEND_ENTRY_ADDR_OK)
+	{
+		if (mFlags & BD_FRIEND_ENTRY_ONLINE)
+		{
+			*addr = mPeerId.addr;
+			return true;
+		}
+
+		
+		if (time(NULL) - mLastSeen < BD_FRIEND_ENTRY_TIMEOUT)
+		{
+			*addr = mPeerId.addr;
+			return true;
+		}
+	}
+	return false;
+}
+
+uint32_t bdFriendEntry::getPeerFlags()
+{
+	return mFlags & BD_FRIEND_ENTRY_MASK_KNOWN;
+}
+
+
+bdFriendList::bdFriendList(const bdNodeId *ownId)
+{
+	bdId tmpId;
+	tmpId.id = *ownId;
+	updatePeer(&tmpId, BD_FRIEND_ENTRY_SELF);
+}
+
+/******
+ * Simple logic: timestamp is set with address.
+ * if ONLINE, then address will be dropped as soon as OFFLINE.
+ * if ADDR_OK, then address will be dropped after XX seconds.
+ *
+ * ONLINE - will be used with friends.
+ * ADDR_OK - will potentially be used for friends of friends (but not for now).
+ *****/
+
+	/* catch-all interface function */
+bool	bdFriendList::updatePeer(const bdId *id, uint32_t flags)
+{
+#ifdef DEBUG_FRIENDLIST	
+	std::cerr << "bdFriendList::updatePeer() Peer(";
+	bdStdPrintId(std::cerr, id);
+	std::cerr << ") Flags: " << flags;
+	std::cerr << std::endl;
+#endif
+
+	std::map<bdNodeId, bdFriendEntry>::iterator it;
+	it = mPeers.find(id->id);
+	if (it == mPeers.end())
+	{
+		bdFriendEntry entry;
+		entry.mPeerId.id = id->id;
+		entry.mFlags = 0;
+		entry.mLastSeen = 0;
+
+		mPeers[id->id] = entry;
+		it = mPeers.find(id->id);
+	}
+
+	/* update all */
+	it->second.mFlags = flags;
+	if (it->second.mFlags & (BD_FRIEND_ENTRY_ADDR_OK | BD_FRIEND_ENTRY_ONLINE))
+	{
+		it->second.mFlags |= BD_FRIEND_ENTRY_ADDR_OK;
+
+		it->second.mPeerId.addr = id->addr;
+		it->second.mLastSeen = time(NULL);
+	}
+	return true;
+}
+
+bool	bdFriendList::removePeer(const bdNodeId *id)
+{
+	/* see if it exists... */
+	std::map<bdNodeId, bdFriendEntry>::iterator it;
+	it = mPeers.find(*id);
+	if (it == mPeers.end())
+	{
+#ifdef DEBUG_FRIENDLIST	
+		std::cerr << "bdFriendList::removeFriend() Peer(";
+		bdStdPrintNodeId(std::cerr, id);
+		std::cerr << ") is unknown!";
+		std::cerr << std::endl;
+#endif
+
+		return false;
+	}
+
+	mPeers.erase(*id);
+
+	return true;
+}
+
+bool	bdFriendList::findPeerEntry(const bdNodeId *id, bdFriendEntry &entry)
+{
+	/* see if it exists... */
+	std::map<bdNodeId, bdFriendEntry>::iterator it;
+	it = mPeers.find(*id);
+	if (it == mPeers.end())
+	{
+#ifdef DEBUG_FRIENDLIST
+		std::cerr << "bdFriendList::getPeerEntry() Peer(";
+		bdStdPrintNodeId(std::cerr, id);
+		std::cerr << ") is unknown!";
+		std::cerr << std::endl;
+#endif
+
+		return false;
+	}
+
+	entry = it->second;
+
+
+	return true;
+}
+
+
+bool    bdFriendList::findPeersWithFlags(uint32_t flags, std::list<bdNodeId> &peerList)
+{
+#ifdef DEBUG_FRIENDLIST
+	std::cerr << "bdFriendList::findPeersWithFlags(" << flags << ")";
+	std::cerr << std::endl;
+#endif
+
+	/* see if it exists... */
+	std::map<bdNodeId, bdFriendEntry>::iterator it;
+	for(it = mPeers.begin(); it != mPeers.end(); it++)
+	{
+		/* if they have ALL of the flags we specified */
+		if ((it->second.getPeerFlags() & flags) == flags)
+		{
+#ifdef DEBUG_FRIENDLIST
+			std::cerr << "bdFriendList::findPeersWithFlags() Found: ";
+			bdStdPrintNodeId(std::cerr, id);
+			std::cerr << std::endl;
+#endif
+			peerList.push_back(it->second.mPeerId.id);
+		}
+	}
+	return (peerList.size() > 0);
+}
+
+
+
+bool	bdFriendList::print(std::ostream &out)
+{
+	time_t now = time(NULL);
+
+	out << "bdFriendList::print()";
+	out << std::endl;
+
+	std::map<bdNodeId, bdFriendEntry>::iterator it;
+	for(it = mPeers.begin(); it != mPeers.end(); it++)
+	{
+		bdStdPrintId(out, &(it->second.mPeerId));
+		out << " Flags: " << it->second.mFlags;
+		out << " Seen: " << now - it->second.mLastSeen;
+		out << std::endl;
+	}
+
+	return true;
+}
+
+
+
+
+bdPeerQueue::bdPeerQueue()
+{
+	return;
+}
+
+bool bdPeerQueue::queuePeer(const bdId *id, uint32_t flags)
+{
+	bdFriendEntry entry;
+	entry.mPeerId = *id;
+	entry.mFlags = flags;
+	entry.mLastSeen = time(NULL);
+
+	mPeerQueue.push_back(entry);
+	return true;
+}
+
+bool bdPeerQueue::popPeer(bdId *id, uint32_t &flags)
+{
+	if (mPeerQueue.size() > 0)
+	{
+		bdFriendEntry entry = mPeerQueue.front();
+		mPeerQueue.pop_front();
+		*id = entry.mPeerId;
+		flags = entry.mFlags;
+
+		return true;
+	}
+
+	return false;
+}
+
+
+

libbitdht/src/bitdht/bdfriendlist.h

@@ -0,0 +1,101 @@
+/*******************************************************************************
+ * bitdht/bdfriendlist.h                                                       *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#ifndef BITDHT_FRIEND_LIST_H
+#define BITDHT_FRIEND_LIST_H
+
+/* 
+ * This class maintains a list of current friends and friends-of-friends.
+ * It should also be updated when a peer's address has been identified.
+ *
+ * It is used for selecting preferential peers for DHT Connections.
+ * and for detecting bad peers that are duplicating real RS peers.
+ *
+ */
+
+
+#include "bitdht/bdiface.h"
+#include <set>
+
+#define BD_FRIEND_ENTRY_TIMEOUT		10
+
+
+//#define BD_FRIEND_ENTRY_ONLINE		0x0001
+//#define BD_FRIEND_ENTRY_ADDR_OK		0x0002
+
+//#define BD_FRIEND_ENTRY_WHITELIST	BITDHT_PEER_STATUS_DHT_WHITELIST
+//#define BD_FRIEND_ENTRY_FOF		BITDHT_PEER_STATUS_DHT_FOF
+//#define BD_FRIEND_ENTRY_FRIEND 	BITDHT_PEER_STATUS_DHT_FRIEND
+//#define BD_FRIEND_ENTRY_RELAY_SERVER	BITDHT_PEER_STATUS_DHT_RELAY_SERVER
+
+//#define BD_FRIEND_ENTRY_SELF		BITDHT_PEER_STATUS_DHT_SELF
+
+//#define BD_FRIEND_ENTRY_MASK_KNOWN	BITDHT_PEER_STATUS_MASK_KNOWN
+
+class bdFriendEntry
+{
+	public:
+	bdFriendEntry();
+
+bool	addrKnown(struct sockaddr_in *addr);
+uint32_t getPeerFlags();
+
+	bdId mPeerId;
+	uint32_t mFlags; 
+	time_t mLastSeen;
+};
+
+class bdFriendList
+{
+
+	public:
+	bdFriendList(const bdNodeId *ownid);
+
+bool	updatePeer(const bdId *id, uint32_t flags);
+bool	removePeer(const bdNodeId *id);
+
+bool	findPeerEntry(const bdNodeId *id, bdFriendEntry &entry);
+bool    findPeersWithFlags(uint32_t flags, std::list<bdNodeId> &peerList);
+
+bool    print(std::ostream &out);
+	private:
+
+	std::map<bdNodeId, bdFriendEntry> mPeers;
+};
+
+class bdPeerQueue
+{
+
+	public:
+	bdPeerQueue();
+
+bool	queuePeer(const bdId *id, uint32_t flags);
+bool	popPeer(bdId *id, uint32_t &flags);
+
+	private:
+
+	std::list<bdFriendEntry> mPeerQueue;
+};
+
+	
+#endif
+

libbitdht/src/bitdht/bdhash.cc

@@ -0,0 +1,283 @@
+/*******************************************************************************
+ * bitdht/bdhash.cc                                                            *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "bitdht/bdhash.h"
+#include "bitdht/bdstddht.h"
+#include <iostream>
+
+bdHashEntry::bdHashEntry(std::string value, std::string secret, time_t lifetime, time_t store)
+	:mValue(value), mStoreTS(store), mSecret(secret), mLifetime(lifetime)
+{
+	return;
+}
+
+
+/**************************** class bdHashSet ********************************/
+
+bdHashSet::bdHashSet(bdNodeId *id)
+	:mId(*id)
+{
+	return;
+}
+
+int 	bdHashSet::search(std::string key, uint32_t maxAge, std::list<bdHashEntry> &entries)
+{
+	std::multimap<std::string, bdHashEntry>::iterator sit, eit, it;
+        sit = mEntries.lower_bound(key);
+        eit = mEntries.upper_bound(key);
+
+	time_t now = time(NULL);
+
+	for(it = sit; it != eit; it++)
+	{
+		time_t age = now - it->second.mStoreTS;
+		if (age < (int32_t) maxAge)
+		{
+			entries.push_back(it->second);
+		}
+	}
+	return (0 < entries.size());
+}
+
+/*** 
+ * With modification.
+ * If there is no secret -> it cannot be deleted (must timeout), but can be extended.
+ * If there is a secret -> must include it to modify.
+ *
+ * Therefore if identical entry without secret comes along - what do I do?
+ * -> create duplicate?
+ */
+
+int 	bdHashSet::modify(std::string key, bdHashEntry *entry, uint32_t modFlags)
+{
+	std::multimap<std::string, bdHashEntry>::iterator sit, eit, it;
+        sit = mEntries.lower_bound(key);
+        eit = mEntries.upper_bound(key);
+
+	time_t now = time(NULL);
+
+	bool updated = false;
+	for(it = sit; it != eit; it++)
+	{
+		/* check it all */
+		if (it->second.mValue == entry->mValue)
+		{
+			bool noSecret = (it->second.mSecret == "");
+			bool sameSecret = (it->second.mSecret == entry->mSecret);
+			bool update = false;
+
+			if (noSecret && sameSecret)
+			{
+				/* only allowed to increase lifetime */
+				if (modFlags == BITDHT_HASH_ENTRY_ADD)
+				{
+					time_t existKillTime = it->second.mLifetime + it->second.mStoreTS;
+					time_t newKillTime = entry->mLifetime + now;
+					if (newKillTime > existKillTime)
+					{
+						update = true;
+					}
+				}
+			}
+			else if (sameSecret)
+			{
+				if (modFlags == BITDHT_HASH_ENTRY_ADD)
+				{
+					update = true;
+				}
+				else if (modFlags == BITDHT_HASH_ENTRY_DELETE)
+				{
+					/* do it here */
+					mEntries.erase(it);
+					return 1;
+				}
+			}
+			
+			if (update)
+			{
+				it->second.mStoreTS = now;
+				it->second.mLifetime = entry->mLifetime;
+				updated = true;
+			}
+		}
+	}
+
+	if ((!updated) && (modFlags == BITDHT_HASH_ENTRY_ADD))
+	{
+		/* create a new entry */
+		bdHashEntry newEntry(entry->mValue, entry->mSecret, entry->mLifetime, now);
+	        mEntries.insert(std::pair<std::string, bdHashEntry>(key, newEntry));
+		updated = true;
+	}
+	return updated;
+}
+
+int	bdHashSet::printHashSet(std::ostream &out)
+{
+	time_t now = time(NULL);
+	std::multimap<std::string, bdHashEntry>::iterator it;
+	out << "Hash: ";
+	bdStdPrintNodeId(out, &mId); // Allowing "Std" as we dont need dht functions everywhere.
+	out << std::endl;
+
+	for(it = mEntries.begin(); it != mEntries.end();it++)
+	{
+		time_t age = now - it->second.mStoreTS;
+		out << "\tK:" << bdStdConvertToPrintable(it->first);
+		out << " V:" << bdStdConvertToPrintable(it->second.mValue);
+		out << " A:" << age << " L:" << it->second.mLifetime;
+		out << " S:" << bdStdConvertToPrintable(it->second.mSecret);
+		out << std::endl;
+	}
+	return 1;
+}
+
+	
+	
+
+int	bdHashSet::cleanupHashSet(uint32_t maxAge)
+{
+	time_t now = time(NULL);
+
+	/* this is nasty... but don't know how to effectively remove from multimaps
+	 * * Must do full repeat for each removal.
+	 */
+
+	std::multimap<std::string, bdHashEntry>::iterator it;
+	for(it = mEntries.begin(); it != mEntries.end();)
+	{
+		time_t age = now - it->second.mStoreTS;
+		if ((age > (int32_t) maxAge) || (age > it->second.mLifetime))
+		{
+			mEntries.erase(it);
+			it = mEntries.begin();
+		}
+		else
+		{
+			it++;
+		}
+	}
+	return 1;
+}
+
+
+	
+
+
+
+/*******************************  class bdHashSpace ***************************/
+
+bdHashSpace::bdHashSpace()
+{
+	return;
+}
+
+	/* accessors */
+int 	bdHashSpace::search(bdNodeId *id, std::string key, uint32_t maxAge, std::list<bdHashEntry> &entries)
+{
+	std::map<bdNodeId, bdHashSet>::iterator it;
+	it = mHashTable.find(*id);
+	if (it == mHashTable.end()) 
+	{
+		/* no entry */
+		return 1;
+	}
+
+	return it->second.search(key, maxAge, entries);
+}
+
+int 	bdHashSpace::modify(bdNodeId *id, std::string key, bdHashEntry *entry, uint32_t modFlags)
+{
+	std::map<bdNodeId, bdHashSet>::iterator it;
+	it = mHashTable.find(*id);
+	if (it == mHashTable.end()) 
+	{
+		if (modFlags == BITDHT_HASH_ENTRY_DELETE)
+		{
+			/* done already */
+			return 1;
+		}
+			
+		//mHashTable[*id] = bdHashSet(id);
+	        mHashTable.insert(std::pair<bdNodeId, bdHashSet>(*id, bdHashSet(id)));
+		it = mHashTable.find(*id);
+	}
+
+	return it->second.modify(key, entry, modFlags);
+}
+
+int     bdHashSpace::printHashSpace(std::ostream &out)
+{
+	std::map<bdNodeId, bdHashSet>::iterator it;
+	out << "bdHashSpace::printHashSpace()" << std::endl;
+	out << "--------------------------------------------" << std::endl;
+	
+	for(it = mHashTable.begin(); it != mHashTable.end(); it++)
+	{
+		it->second.printHashSet(out);
+	}
+	out << "--------------------------------------------" << std::endl;
+	return 1;
+}
+
+int     bdHashSpace::cleanHashSpace(bdNodeId *min, bdNodeId *max, time_t maxAge)
+{
+	std::map<bdNodeId, bdHashSet>::iterator it;
+	std::list<bdNodeId> eraseList;
+	std::list<bdNodeId>::iterator eit;
+
+	for(it = mHashTable.begin(); it != mHashTable.end(); it++)
+	{
+		if ((it->first < *min) ||
+			(*max < it->first))
+		{
+			/* schedule for erasure */
+			eraseList.push_back(it->first);
+		}
+		else
+		{
+			/* clean up Hash Set */
+			it->second.cleanupHashSet(maxAge);	
+		}
+	}
+
+	/* cleanup */
+	while(eraseList.size() > 0)
+	{
+		bdNodeId &eId = eraseList.front();
+		it = mHashTable.find(eId);
+		if (it != mHashTable.end())
+		{
+			mHashTable.erase(it);
+		}
+	}
+	return 1;
+}
+
+
+int     bdHashSpace::clear()
+{
+	mHashTable.clear();
+	return 1;
+}
+
+

libbitdht/src/bitdht/bdhash.h

@@ -0,0 +1,85 @@
+/*******************************************************************************
+ * bitdht/bdhash.h                                                             *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#ifndef BITDHT_HASH_SPACE_H
+#define BITDHT_HASH_SPACE_H
+
+#include "bitdht/bdpeer.h"
+
+#include <list>
+#include <string>
+#include <map>
+
+#define BITDHT_HASH_ENTRY_ADD		1
+#define BITDHT_HASH_ENTRY_DELETE	2
+
+class bdHashEntry
+{
+	public:
+
+	bdHashEntry(std::string value, std::string secret, time_t lifetime, time_t store);
+	std::string mValue;
+	time_t mStoreTS;
+
+	/* These are nice features that OpenDHT had */
+	std::string mSecret;
+	time_t mLifetime;
+};
+
+
+class bdHashSet
+{
+	public:
+
+	bdHashSet(bdNodeId *id);
+
+int     search(std::string key, uint32_t maxAge, std::list<bdHashEntry> &entries);
+int     modify(std::string key, bdHashEntry *entry, uint32_t modFlags);
+int     printHashSet(std::ostream &out);
+int     cleanupHashSet(uint32_t maxAge);
+
+	bdNodeId mId;
+	std::multimap<std::string, bdHashEntry> mEntries;
+};
+
+class bdHashSpace
+{
+	public:
+
+	bdHashSpace();
+
+	/* accessors */
+int 	search(bdNodeId *id, std::string key, uint32_t maxAge, std::list<bdHashEntry> &entries);
+int 	modify(bdNodeId *id, std::string key, bdHashEntry *entry, uint32_t modFlags);
+
+int 	printHashSpace(std::ostream&);
+int     cleanHashSpace(bdNodeId *min, bdNodeId *max, time_t maxAge);
+int     clear();
+
+	private:
+
+	std::map<bdNodeId, bdHashSet> mHashTable;
+};
+
+
+#endif
+

libbitdht/src/bitdht/bdhistory.cc

@@ -0,0 +1,647 @@
+/*******************************************************************************
+ * bitdht/bdhistory.cc                                                         *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+
+#include "bitdht/bdhistory.h"
+#include "bitdht/bdstddht.h"
+#include "bitdht/bdmsgs.h"
+#include <set>
+
+#define MIN_RESEND_PERIOD  60
+
+bdMsgHistoryList::bdMsgHistoryList()
+:mPeerVersion("Unknown")
+{
+	return;
+}
+
+
+
+void	bdMsgHistoryList::addMsg(time_t ts, uint32_t msgType, bool incoming, const bdNodeId *aboutId)
+{
+//	std::cerr << "bdMsgHistoryList::addMsg()";
+//	std::cerr << std::endl;
+
+	bdMsgHistoryItem msg(msgType, incoming, aboutId);
+	msgHistory.insert(std::make_pair(ts, msg));
+}
+
+void 	bdMsgHistoryList::setPeerType(time_t /* ts */, std::string version)
+{
+	mPeerVersion = version;
+}
+
+int	bdMsgHistoryList::msgCount(time_t start_ts, time_t end_ts)
+{
+	std::multimap<time_t, bdMsgHistoryItem>::iterator sit, eit, it;
+	sit = msgHistory.lower_bound(start_ts);
+	eit = msgHistory.upper_bound(end_ts);
+	int count = 0;
+	for (it = sit; it != eit; it++, count++) ; // empty loop.
+
+	return count;
+}
+
+bool    bdMsgHistoryList::msgClear(time_t before)
+{
+	if (before == 0)
+	{
+		msgHistory.clear();
+		return true;
+	}
+
+	// Delete the old stuff in the list.
+	while((msgHistory.begin() != msgHistory.end()) && (msgHistory.begin()->first < before))
+	{
+		msgHistory.erase(msgHistory.begin());
+	}
+
+	// return true if empty.
+	if (msgHistory.begin() == msgHistory.end())
+	{
+		return true;
+	}
+	return false;
+}
+
+void	bdMsgHistoryList::msgClear()
+{
+	msgHistory.clear();
+}
+
+
+void	bdMsgHistoryList::clearHistory()
+{
+	msgClear();
+}
+
+void	bdMsgHistoryList::printHistory(std::ostream &out, int mode, time_t start_ts, time_t end_ts)
+{
+	//out << "AGE: MSGS  => incoming, <= outgoing"  << std::endl;
+	std::multimap<time_t, bdMsgHistoryItem>::iterator sit, eit, it;
+	sit = msgHistory.lower_bound(start_ts);
+	eit = msgHistory.upper_bound(end_ts);
+	time_t curr_ts = 0;
+	bool time_changed = false;
+	bool first_line = true;
+	
+	for(it = sit; it != eit; it++)
+	{
+		time_changed = false;
+		if (curr_ts != it->first)
+		{
+			curr_ts = it->first;
+			time_changed = true;
+		}
+		switch(mode)
+		{
+			default:
+		{
+			/* print one line per ts */
+			if (time_changed)
+			{
+				if (!first_line)
+				{
+					/* finish existing line */
+					out << " " << std::endl;
+				}
+				else
+				{
+					first_line = false;
+				}
+				out << "\tTS: " << time(NULL) - curr_ts << " ";
+			}
+	
+			std::string name;
+			bitdht_msgtype(it->second.msgType, name);
+
+			if (it->second.incoming)
+			{
+				out << "( =I> ";
+			}
+			else
+			{
+				out << "( <O= ";
+			}
+
+			out << name << " ";
+			if ((it->second.aboutId.data[0] == 0)
+				&& (it->second.aboutId.data[3] == 0)
+				&& (it->second.aboutId.data[3] == 0)
+				&& (it->second.aboutId.data[3] == 0))
+			{
+				/* don't print anything */
+			}
+			else
+			{
+				bdStdPrintNodeId(out, &(it->second.aboutId));
+			}
+			out << " )";
+
+		}
+			break;
+		} // end of switch.
+	}
+
+	/* finish up last line */
+	if (!first_line)
+	{
+		out << " " << std::endl;
+	}
+}
+
+
+bool	bdMsgHistoryList::canSend()
+{
+	std::cerr << "bdMsgHistoryList::canSend()";
+
+	std::multimap<time_t, bdMsgHistoryItem>::reverse_iterator rit;
+
+	rit = msgHistory.rbegin();
+	if (rit != msgHistory.rend())
+	{
+		time_t now = time(NULL);
+		if (now - rit->first > MIN_RESEND_PERIOD)
+		{
+			std::cerr << " OVER RESEND_PERIOD... true";
+			std::cerr << std::endl;
+
+			return true;
+		}
+	}
+
+	if (msgHistory.size() % 2 == 0)
+	{
+		std::cerr << " SIZE: " << msgHistory.size() << " % 2 = 0 ... true";
+		std::cerr << std::endl;
+
+		return true;
+	}
+
+	std::cerr << " false";
+	std::cerr << std::endl;
+
+	return false;
+}
+
+bool	bdMsgHistoryList::validPeer()
+{
+	std::cerr << "bdMsgHistoryList::validPeer()";
+
+	std::multimap<time_t, bdMsgHistoryItem>::iterator it;
+
+	for(it = msgHistory.begin(); it != msgHistory.end(); it++)
+	{
+		if (it->second.incoming)
+		{
+			std::cerr << " Incoming Msg... so validPeer";
+			std::cerr << std::endl;
+
+			return true;
+		}
+	}
+
+	std::cerr << " false";
+	std::cerr << std::endl;
+
+	return false;
+}
+
+
+#define MAX_PING_PER_MINUTE  	2
+#define MAX_QUERY_PER_MINUTE 	2
+
+bool bdMsgHistoryList::analysePeer()
+
+{
+	/* analyse and print out details of the peers messages */
+	bool flagged = false;
+
+	//out << "AGE: MSGS  => incoming, <= outgoing"  << std::endl;
+	std::multimap<time_t, bdMsgHistoryItem>::iterator sit, eit, it;
+	sit = msgHistory.begin();
+	eit = msgHistory.end();
+	if (sit == eit)
+	{
+		// nothing here.
+		return false;
+	}
+
+	time_t start_ts = sit->first;
+	time_t end_ts = msgHistory.rbegin()->first; // must exist.
+
+
+	// don't divide by zero.
+	if (end_ts - start_ts < 60)
+	{
+		end_ts = start_ts + 60;
+	}
+
+
+	/* what do we want to analyse? */
+
+	/* if we have sent / recved too many queries or pings */
+
+	int in_ping = 0;
+	int out_ping = 0;
+	int in_query = 0;
+	int out_query = 0;
+	int in_other = 0;
+	int out_other = 0;
+
+	for(it = sit; it != eit; it++)
+	{
+		if (it->second.incoming)
+		{
+			switch(it->second.msgType)
+			{
+				case BITDHT_MSG_TYPE_PING:
+					in_ping++;
+					break;
+				case BITDHT_MSG_TYPE_FIND_NODE:
+					in_query++;
+					break;
+				default:
+					in_other++;
+					break;
+			}
+		}
+		else
+		{
+			switch(it->second.msgType)
+			{
+				case BITDHT_MSG_TYPE_PING:
+					out_ping++;
+					break;
+				case BITDHT_MSG_TYPE_FIND_NODE:
+					out_query++;
+					break;
+				default:
+					out_other++;
+					break;
+			}
+		}
+
+	}
+
+	float in_ping_per_min = in_ping * 60.0 / (end_ts - start_ts);
+	float out_ping_per_min = out_ping * 60.0 / (end_ts - start_ts);
+	float in_query_per_min = in_query * 60.0 / (end_ts - start_ts);
+	float out_query_per_min = out_query * 60.0 / (end_ts - start_ts);
+
+	if ((in_ping_per_min > MAX_PING_PER_MINUTE) ||
+		(out_ping_per_min > MAX_PING_PER_MINUTE) ||
+		(in_query_per_min > MAX_PING_PER_MINUTE) ||
+		(out_query_per_min > MAX_PING_PER_MINUTE))
+	{
+		flagged = true;
+	}
+
+	if (flagged)
+	{
+		/* print header */
+		std::ostream &out = std::cerr;
+		out << "BdHistoryAnalysis has flagged peer: ";
+		bdStdPrintId(out, &mId);
+		out << std::endl;
+
+		out << "PeerType: " << mPeerVersion;
+		out << std::endl;
+
+		out << "Ping In Per Min  : " << in_ping_per_min << std::endl;
+		out << "Ping Out Per Min : " << out_ping_per_min << std::endl;
+		out << "Query In Per Min : " << in_query_per_min << std::endl;
+		out << "Query Out Per Min: " << out_query_per_min << std::endl;
+
+		out << "Message History: ";
+		out << std::endl;
+
+		printHistory(out, 0, 0, time(NULL));
+	}
+	return true;
+}
+
+bdHistory::bdHistory(time_t store_period)
+ :mStorePeriod(store_period) { return; }
+
+void bdHistory::addMsg(const bdId *id, bdToken * /*transId*/, uint32_t msgType, bool incoming, const bdNodeId *aboutId)
+{
+	//std::cerr << "bdHistory::addMsg() ";
+	//bdStdPrintId(std::cerr, id);
+	//std::cerr << std::endl;
+
+	time_t now = time(NULL);
+
+	std::map<bdId, bdMsgHistoryList>::iterator it;
+	bdMsgHistoryList &histRef = mHistory[*id]; /* will instaniate empty */
+	histRef.mId = *id;
+	histRef.addMsg(now, msgType, incoming, aboutId);
+
+	/* add to mMsgTimeline */
+	mMsgTimeline.insert(std::make_pair(now, MsgRegister(id, msgType, incoming, aboutId)));
+}
+
+void bdHistory::setPeerType(const bdId *id, std::string version)
+{
+	std::map<bdId, bdMsgHistoryList>::iterator it;
+	bdMsgHistoryList &histRef = mHistory[*id]; /* will instaniate empty */
+	histRef.setPeerType(time(NULL), version);
+}
+
+void bdHistory::printMsgs()
+{
+	/* print and clear msgs */
+	std::ostream &out = std::cerr;
+
+	std::cerr << "bdHistory::printMsgs()";
+	std::cerr << std::endl;
+
+
+	std::map<bdId, bdMsgHistoryList> ::iterator it;
+	for(it = mHistory.begin(); it != mHistory.end(); it++)
+	{
+		if (it->second.msgCount(0, time(NULL))) // all msg count.
+		{
+			/* print header */
+			out << "Msgs for ";
+			bdStdPrintId(out, &(it->first));
+			out << " v:" << it->second.mPeerVersion;
+			out << std::endl;
+
+			it->second.printHistory(out, 0, 0, time(NULL));
+		}
+	}
+
+	out << "Msg Timeline:";
+	time_t now = time(NULL);
+	std::multimap<time_t, MsgRegister>::iterator hit;
+	for(hit = mMsgTimeline.begin(); hit != mMsgTimeline.end(); hit++)
+	{
+		out << now - hit->first << "   ";
+		bdStdPrintId(out, &(hit->second.id));
+
+		if (hit->second.incoming)
+		{
+			out << " =I> ";
+		}
+		else
+		{
+			out << " <O= ";
+		}
+
+		std::string name;
+		if (bitdht_msgtype(hit->second.msgType, name))
+		{
+			out << name;
+		}
+		else
+		{
+			out << "UNKNOWN MSG";
+		}
+		out << std::endl;
+	}
+}
+
+
+void bdHistory::cleanupOldMsgs()
+{
+	std::cerr << "bdHistory::cleanupOldMsgs()";
+	std::cerr << std::endl;
+
+	if (mStorePeriod == 0)
+	{
+		return; // no cleanup
+	}
+
+	std::set<bdId> to_cleanup;
+	std::set<bdId>::iterator cit;
+
+	time_t before = time(NULL) - mStorePeriod;
+
+	// Delete the old stuff in the list.
+	while((mMsgTimeline.begin() != mMsgTimeline.end()) && (mMsgTimeline.begin()->first < before))
+	{
+		std::multimap<time_t, MsgRegister>::iterator it = mMsgTimeline.begin();
+		to_cleanup.insert(it->second.id);
+		mMsgTimeline.erase(it);
+	}
+
+	// remove old msgs, delete entry if its empty.
+	std::map<bdId, bdMsgHistoryList>::iterator hit;
+	for(cit = to_cleanup.begin(); cit != to_cleanup.end(); cit++)
+	{
+		hit = mHistory.find(*cit);
+		if (hit != mHistory.end())
+		{
+			if (hit->second.msgClear(before))
+			{
+				// don't erase actual entry (so we remember peer type).
+				//mHistory.erase(hit);
+			}
+		}
+	}
+}
+
+
+
+void bdHistory::clearHistory()
+{
+	// Switched to a alternative clear, so we don't drop peers, and remember their type.
+	//mHistory.clear();
+	
+
+	std::map<bdId, bdMsgHistoryList> ::iterator it;
+	for(it = mHistory.begin(); it != mHistory.end(); it++)
+	{
+		it->second.clearHistory();
+	}
+}
+
+bool bdHistory::canSend(const bdId *id)
+{
+
+	std::map<bdId, bdMsgHistoryList> ::iterator it;
+	it = mHistory.find(*id);
+	if (it != mHistory.end())
+	{
+		return (it->second.canSend());
+	}
+
+	/* if not found - then can send */
+	return true;
+}
+
+
+bool bdHistory::validPeer(const bdId *id)
+{
+	std::map<bdId, bdMsgHistoryList> ::iterator it;
+	it = mHistory.find(*id);
+	if (it != mHistory.end())
+	{
+		return (it->second.validPeer());
+	}
+
+	/* if not found - then can send */
+	return false;
+}
+
+bool bdHistory::analysePeers()
+{
+	std::map<bdId, bdMsgHistoryList> ::iterator it;
+	for(it = mHistory.begin(); it != mHistory.end(); it++)
+	{
+		it->second.analysePeer();
+	}
+	return true;
+}
+
+
+
+/* Temp data class. */
+class TypeStats
+{
+	public:
+
+	TypeStats() :nodes(0) { return; }
+
+	std::map<uint32_t, uint32_t> incoming, outgoing;
+	int nodes;
+
+
+void	printStats(std::ostream &out, const TypeStats *refStats)
+{
+	std::map<uint32_t, uint32_t>::iterator it;
+	std::map<uint32_t, uint32_t>::const_iterator rit;
+
+	out << "  Nodes: " << nodes;
+	if (refStats)
+	{
+		out << " (" << 100.0 * nodes / (float) refStats->nodes << " %)";
+	}
+	out << std::endl;
+
+	out << "  Incoming Msgs";
+	out << std::endl;
+	for(it = incoming.begin(); it != incoming.end(); it++)
+	{
+		uint32_t count = 0;
+		if (refStats)
+		{			
+			rit = refStats->incoming.find(it->first);
+			if (rit != refStats->incoming.end())
+			{
+				count = rit->second;
+			}
+		}
+		printStatsLine(out, it->first, it->second, count);
+	}
+
+	out << "  Outgoing Msgs";
+	out << std::endl;
+	for(it = outgoing.begin(); it != outgoing.end(); it++)
+	{
+		uint32_t count = 0;
+		if (refStats)
+		{			
+			rit = refStats->outgoing.find(it->first);
+			if (rit != refStats->outgoing.end())
+			{
+				count = rit->second;
+			}
+		}
+		printStatsLine(out, it->first, it->second, count);
+	}
+}
+
+
+
+void   printStatsLine(std::ostream &out, uint32_t msgType, uint32_t count, uint32_t global)
+{
+	std::string name;
+	bitdht_msgtype(msgType, name);
+	out << "\t" << name << " " << count;
+	if (global != 0)
+	{
+		out << " (" << 100.0 * count / (float) global << " %)";
+	}
+	out << std::endl;
+}
+
+}; /* end of TypeStats */
+
+bool bdHistory::peerTypeAnalysis()
+{
+
+	std::map<std::string, TypeStats> mTypeStats;
+	TypeStats globalStats;
+
+	std::map<bdId, bdMsgHistoryList> ::iterator it;
+	for(it = mHistory.begin(); it != mHistory.end(); it++)
+	{
+		if (it->second.msgHistory.empty())
+		{
+			continue;
+		}
+
+		std::string version = it->second.mPeerVersion;
+		// group be first two bytes.
+		version = it->second.mPeerVersion.substr(0,2);
+		TypeStats &stats = mTypeStats[version];
+
+		stats.nodes++;
+		globalStats.nodes++;
+
+		std::multimap<time_t, bdMsgHistoryItem>::iterator lit;
+		for (lit = it->second.msgHistory.begin(); lit != it->second.msgHistory.end(); lit++)
+		{
+			if (lit->second.incoming)
+			{
+				stats.incoming[lit->second.msgType]++;
+				globalStats.incoming[lit->second.msgType]++;
+			}
+			else
+			{
+				stats.outgoing[lit->second.msgType]++;
+				globalStats.outgoing[lit->second.msgType]++;
+			}
+		}
+	}
+
+
+	std::map<std::string, TypeStats>::iterator tit;
+	for(tit = mTypeStats.begin(); tit != mTypeStats.end(); tit++)
+	{
+		std::cerr << "Stats for Peer Type: " << tit->first;
+		std::cerr << std::endl;
+		tit->second.printStats(std::cerr, &globalStats);	
+	}
+
+	std::cerr << "Global Stats: ";
+	std::cerr << std::endl;
+
+	globalStats.printStats(std::cerr, NULL);
+	return true;
+
+}
+
+
+

libbitdht/src/bitdht/bdhistory.h

@@ -0,0 +1,147 @@
+/*******************************************************************************
+ * bitdht/bdhistory.h                                                          *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#ifndef BITDHT_HISTORY_H
+#define BITDHT_HISTORY_H
+
+#include "bitdht/bdpeer.h"
+#include "bitdht/bdobj.h"
+#include "bitdht/bdstddht.h"
+#include <map>
+
+#define MSG_TYPE_DIRECTION_MASK	0x000f0000
+
+#define MSG_DIRECTION_INCOMING  0x00010000
+#define MSG_DIRECTION_OUTGOING  0x00020000
+
+/**** DEBUGGING HISTORY ****/
+
+class MsgRegister
+{
+        public:
+        MsgRegister() { return; }
+        MsgRegister(const bdId *inId, uint32_t inMsgType, bool inIncoming, const bdNodeId *inAboutId)
+          :id(*inId), msgType(inMsgType), incoming(inIncoming) 
+	{ 
+		if (inAboutId)
+		{
+			aboutId = *inAboutId;
+		}
+		else
+		{
+			bdStdZeroNodeId(&aboutId);
+		}
+
+		return; 
+	}
+
+        bdId id;
+        uint32_t msgType;
+        bool incoming;
+	bdNodeId aboutId; // filled in for queries.
+};
+
+
+class bdMsgHistoryItem
+{
+	public:
+	bdMsgHistoryItem() 
+          :msgType(0), incoming(false) 
+	{
+		bdStdZeroNodeId(&aboutId);
+		return;
+	}
+
+        bdMsgHistoryItem(uint32_t inMsgType, bool inIncoming, const bdNodeId *inAboutId)
+          :msgType(inMsgType), incoming(inIncoming) 
+	{ 
+		if (inAboutId)
+		{
+			aboutId = *inAboutId;
+		}
+		else
+		{
+			bdStdZeroNodeId(&aboutId);
+		}
+
+		return; 
+	}
+
+	uint32_t msgType;
+	bool incoming;
+	bdNodeId aboutId; // filled in for queries.
+};
+
+
+class bdMsgHistoryList
+{
+	public:
+	bdMsgHistoryList();
+void	addMsg(time_t ts, uint32_t msgType, bool incoming, const bdNodeId *aboutId);
+void 	setPeerType(time_t ts, std::string version);
+int	msgCount(time_t start_ts, time_t end_ts);
+bool    msgClear(time_t before); // 0 => clear all.
+void	msgClear();
+void	printHistory(std::ostream &out, int mode, time_t start_ts, time_t end_ts);
+bool    analysePeer();
+void    clearHistory();
+
+bool    canSend();
+bool    validPeer();
+
+	std::multimap<time_t, bdMsgHistoryItem> msgHistory;
+	std::string mPeerVersion;
+	bdId mId;
+};
+
+
+
+class bdHistory
+{
+	public:
+	bdHistory(time_t store_period);
+
+void	addMsg(const bdId *id, bdToken *transId, uint32_t msgType, bool incoming, const bdNodeId *aboutId);
+void 	setPeerType(const bdId *id, std::string version);
+void	printMsgs();
+
+void    cleanupOldMsgs();
+void    clearHistory();
+bool    analysePeers();
+bool 	peerTypeAnalysis();
+
+bool    canSend(const bdId *id);
+bool    validPeer(const bdId *id);
+
+	/* recent history */
+	//std::list<bdId> lastMsgs;
+	std::map<bdId, bdMsgHistoryList> mHistory;
+	std::multimap<time_t, MsgRegister> mMsgTimeline;
+
+	int mStorePeriod;
+
+};
+
+
+
+#endif
+

libbitdht/src/bitdht/bdiface.h

@@ -0,0 +1,413 @@
+/*******************************************************************************
+ * bitdht/bdiface.h                                                            *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#ifndef BIT_DHT_INTERFACE_H
+#define BIT_DHT_INTERFACE_H
+
+#include <iosfwd>
+#include <map>
+#include <string>
+#include <list>
+#include <inttypes.h>
+
+#include "util/bdnet.h"
+
+/*
+ * Functions and Classes required for Interfacing with the BitDht.
+ * This should be the sole header file required to talk to Dht.
+ * ... though setting it up will require including udpbitdht.h as well.
+ * 
+ */
+
+#define BITDHT_KEY_LEN 20
+#define BITDHT_KEY_INTLEN 5
+#define BITDHT_KEY_BITLEN 160
+
+
+
+
+#define BITDHT_MAX_PKTSIZE 1024
+
+#define BITDHT_TTL 64
+
+#define BITDHT_SEARCH_ONE_SHOT 		1
+#define BITDHT_SEARCH_REPEATING		2
+
+class bdNodeId
+{
+        public:
+        unsigned char data[BITDHT_KEY_LEN];
+};
+
+class bdMetric: public bdNodeId {};
+
+class bdId
+{
+        public:
+
+        bdId();
+        bdId(bdNodeId in_id, struct sockaddr_in in_addr);
+
+        struct sockaddr_in addr;
+        bdNodeId id;
+};
+
+#define BITDHT_LIKELY_SAME_NO			0x00000000
+#define BITDHT_LIKELY_SAME_YES			0x00000001
+#define BITDHT_LIKELY_SAME_PORT_CHANGED		0x00000002
+#define BITDHT_LIKELY_SAME_LOC_CHANGED		0x00000004
+#define BITDHT_LIKELY_SAME_IDENTICAL		0x00000008
+
+
+class bdDhtFunctions
+{
+	public:
+
+//	bdDhtFunctions();
+	/* setup variables */
+virtual uint16_t bdNumBuckets() = 0;
+virtual uint16_t bdNodesPerBucket() = 0; /* used for bdspace */
+virtual uint16_t bdNumQueryNodes() = 0; /* used for queries */
+virtual uint16_t bdBucketBitSize() = 0;
+
+virtual int bdDistance(const bdNodeId *n1, const bdNodeId *n2, bdMetric *metric) = 0;
+virtual int bdBucketDistance(const bdNodeId *n1, const bdNodeId *n2) = 0;
+virtual int bdBucketDistance(const bdMetric *metric) = 0;
+
+virtual bool bdSimilarId(const bdId *id1, const bdId *id2) = 0;
+virtual bool bdUpdateSimilarId(bdId *dest, const bdId *src) = 0;
+
+virtual void bdRandomMidId(const bdNodeId *target, const bdNodeId *other, bdNodeId *mid) = 0;
+
+virtual void bdPrintId(std::ostream &out, const bdId *a) = 0;
+virtual void bdPrintNodeId(std::ostream &out, const bdNodeId *a) = 0;
+
+};
+
+
+
+// DHT MODES
+#define BITDHT_MODE_TRAFFIC_MASK	0x00000f00
+#define BITDHT_MODE_RELAYSERVER_MASK  	0x0000f000
+
+// These are not ORd - only one can apply.
+#define BITDHT_MODE_TRAFFIC_HIGH	0x00000100
+#define BITDHT_MODE_TRAFFIC_MED		0x00000200
+#define BITDHT_MODE_TRAFFIC_LOW		0x00000300
+#define BITDHT_MODE_TRAFFIC_TRICKLE	0x00000400
+#define BITDHT_MODE_TRAFFIC_DEFAULT	BITDHT_MODE_TRAFFIC_LOW
+
+// These are not ORd - only one can apply.
+#define BITDHT_MODE_RELAYSERVERS_IGNORED	0x00001000
+#define BITDHT_MODE_RELAYSERVERS_FLAGGED	0x00002000
+#define BITDHT_MODE_RELAYSERVERS_ONLY		0x00003000
+#define BITDHT_MODE_RELAYSERVERS_SERVER		0x00004000
+
+
+/* NODE OPTIONS */
+#define BITDHT_OPTIONS_MAINTAIN_UNSTABLE_PORT		0x00000001
+#define BITDHT_OPTIONS_ENABLE_RELAYS			0x00000002
+
+
+/* peer flags
+ * order is important!
+ * higher bits = more priority.
+ * BITDHT_PEER_STATUS_RECVPING
+ * BITDHT_PEER_STATUS_RECVPONG
+ * BITDHT_PEER_STATUS_RECVNODES
+ * BITDHT_PEER_STATUS_RECVHASHES
+ * BITDHT_PEER_STATUS_DHT_ENGINE  (dbXXxx)
+ * BITDHT_PEER_STATUS_DHT_APPL    (XXRSxx)
+ * BITDHT_PEER_STATUS_DHT_VERSION (XXxx50)
+ * 
+ */
+
+#define 	BITDHT_PEER_STATUS_MASK_RECVD		0x000000ff
+#define 	BITDHT_PEER_STATUS_MASK_DHT		0x0000ff00
+#define 	BITDHT_PEER_STATUS_MASK_KNOWN		0x00ff0000
+
+#define 	BITDHT_PEER_STATUS_RECV_PING		0x00000001
+#define 	BITDHT_PEER_STATUS_RECV_PONG		0x00000002
+#define 	BITDHT_PEER_STATUS_RECV_NODES		0x00000004
+#define 	BITDHT_PEER_STATUS_RECV_HASHES		0x00000008
+#define 	BITDHT_PEER_STATUS_RECV_CONNECT_MSG	0x00000010
+
+#define 	BITDHT_PEER_STATUS_DHT_ENGINE		0x00000100
+#define 	BITDHT_PEER_STATUS_DHT_ENGINE_VERSION	0x00000200
+#define 	BITDHT_PEER_STATUS_DHT_APPL		0x00000400
+#define 	BITDHT_PEER_STATUS_DHT_APPL_VERSION	0x00000800
+
+#define 	BITDHT_PEER_STATUS_DHT_WHITELIST	0x00010000
+#define 	BITDHT_PEER_STATUS_DHT_FOF		0x00020000
+#define 	BITDHT_PEER_STATUS_DHT_FRIEND		0x00040000
+#define 	BITDHT_PEER_STATUS_DHT_RELAY_SERVER	0x00080000	// (Flag must be enabled)
+#define 	BITDHT_PEER_STATUS_DHT_SELF		0x00100000	
+
+
+// EXTRA FLAGS are our internal thoughts about the peer.
+#define 	BITDHT_PEER_EXFLAG_MASK_BASIC		0x000000ff
+#define 	BITDHT_PEER_EXFLAG_UNSTABLE		0x00000001	// Port changes.
+#define 	BITDHT_PEER_EXFLAG_ATTACHED		0x00000002 	// We will ping in heavily. (if unstable)
+#define 	BITDHT_PEER_EXFLAG_BADPEER		0x00000004 	// For testing, we flag rather than discard.
+
+
+
+
+
+#define 	BITDHT_CONNECT_MODE_DIRECT		0x00000001
+#define 	BITDHT_CONNECT_MODE_PROXY		0x00000002
+#define 	BITDHT_CONNECT_MODE_RELAY		0x00000004
+
+#define 	BITDHT_CONNECT_OPTION_AUTOPROXY		0x00000001
+
+// STATUS CODES. == 0 is okay, != 0 is error.
+#define 	BITDHT_CONNECT_ANSWER_OKAY		0x00000000
+#define 	BITDHT_CONNECT_ERROR_NONE		(BITDHT_CONNECT_ANSWER_OKAY)
+
+#define 	BITDHT_CONNECT_ERROR_MASK_TYPE		0x0000ffff
+#define 	BITDHT_CONNECT_ERROR_MASK_SOURCE	0x00ff0000
+#define 	BITDHT_CONNECT_ERROR_MASK_CRMOVE	0xff000000
+
+#define 	BITDHT_CONNECT_ERROR_SOURCE_START	0x00010000
+#define 	BITDHT_CONNECT_ERROR_SOURCE_MID		0x00020000
+#define 	BITDHT_CONNECT_ERROR_SOURCE_END 	0x00040000
+#define 	BITDHT_CONNECT_ERROR_SOURCE_OTHER 	0x00080000
+
+#define 	BITDHT_CONNECT_ERROR_CRMOVE_FATAL 	0x01000000
+#define 	BITDHT_CONNECT_ERROR_CRMOVE_NOMOREIDS 	0x02000000
+#define 	BITDHT_CONNECT_ERROR_CRMOVE_NEXTID 	0x04000000
+#define 	BITDHT_CONNECT_ERROR_CRMOVE_PAUSED 	0x08000000
+
+// ERROR CODES.
+#define 	BITDHT_CONNECT_ERROR_GENERIC		0x00000001
+#define 	BITDHT_CONNECT_ERROR_PROTOCOL		0x00000002
+#define 	BITDHT_CONNECT_ERROR_TIMEOUT		0x00000003
+#define 	BITDHT_CONNECT_ERROR_TEMPUNAVAIL	0x00000004   // Haven't got ext address yet.
+#define 	BITDHT_CONNECT_ERROR_NOADDRESS		0x00000005   // Can't find the peer in tables.
+#define 	BITDHT_CONNECT_ERROR_UNREACHABLE	0x00000006   // Symmetric NAT
+
+#define 	BITDHT_CONNECT_ERROR_UNSUPPORTED	0x00000007
+#define 	BITDHT_CONNECT_ERROR_OVERLOADED		0x00000008
+#define 	BITDHT_CONNECT_ERROR_AUTH_DENIED	0x00000009
+#define 	BITDHT_CONNECT_ERROR_DUPLICATE		0x0000000a
+
+// These are slightly special ones used for CB_REQUEST
+#define 	BITDHT_CONNECT_ERROR_TOOMANYRETRY	0x0000000b
+#define 	BITDHT_CONNECT_ERROR_OUTOFPROXY		0x0000000c
+#define 	BITDHT_CONNECT_ERROR_USER		0x0000000d
+
+
+/*************/
+// FRIEND_ENTRY_FLAGS... used by updateKnownPeers().
+
+#define BD_FRIEND_ENTRY_ONLINE          0x0001
+#define BD_FRIEND_ENTRY_ADDR_OK         0x0002
+
+#define BD_FRIEND_ENTRY_WHITELIST       BITDHT_PEER_STATUS_DHT_WHITELIST
+#define BD_FRIEND_ENTRY_FOF             BITDHT_PEER_STATUS_DHT_FOF
+#define BD_FRIEND_ENTRY_FRIEND          BITDHT_PEER_STATUS_DHT_FRIEND
+#define BD_FRIEND_ENTRY_RELAY_SERVER    BITDHT_PEER_STATUS_DHT_RELAY_SERVER
+
+#define BD_FRIEND_ENTRY_SELF            BITDHT_PEER_STATUS_DHT_SELF
+
+#define BD_FRIEND_ENTRY_MASK_KNOWN      BITDHT_PEER_STATUS_MASK_KNOWN
+
+
+
+
+/* Definitions of bdSpace Peer and Bucket are publically available, 
+ * so we can expose the bucket entries for the gui.
+ */
+
+class bdPeer
+{
+	public:
+	bdPeer():mPeerFlags(0), mLastSendTime(0), mLastRecvTime(0), mFoundTime(0), mExtraFlags(0) { return; }
+
+	bdId   mPeerId;
+	uint32_t mPeerFlags;
+	time_t mLastSendTime;
+	time_t mLastRecvTime;
+	time_t mFoundTime;     /* time stamp that peer was found */
+
+	uint32_t mExtraFlags;
+};
+	
+class bdBucket
+{
+	public:
+	
+	bdBucket();
+	/* list so we can queue properly */
+	std::list<bdPeer> entries;
+};
+
+class bdQueryStatus
+{
+        public:
+        uint32_t mStatus;
+        uint32_t mQFlags;
+        std::list<bdId> mResults;
+};
+
+class bdQuerySummary
+{
+        public:
+
+        bdNodeId mId;
+        bdMetric mLimit;
+        uint32_t mState;
+        time_t mQueryTS;
+        uint32_t mQueryFlags;
+        int32_t mSearchTime;
+
+        int32_t mQueryIdlePeerRetryPeriod; // seconds between retries.
+
+        // closest peers
+        std::multimap<bdMetric, bdPeer>  mClosest;
+        std::multimap<bdMetric, bdPeer>  mPotentialPeers;
+        std::list<bdPeer>  mProxiesUnknown;
+        std::list<bdPeer>  mProxiesFlagged;
+};
+
+
+
+
+/* Status options */
+#define BITDHT_QUERY_READY		1
+#define BITDHT_QUERY_QUERYING           2
+#define BITDHT_QUERY_FAILURE            3
+#define BITDHT_QUERY_FOUND_CLOSEST      4
+#define BITDHT_QUERY_PEER_UNREACHABLE   5
+#define BITDHT_QUERY_SUCCESS            6
+
+/* Query Flags */
+#define BITDHT_QFLAGS_NONE		0x0000
+#define BITDHT_QFLAGS_DISGUISE		0x0001  // Don't search directly for target.
+#define BITDHT_QFLAGS_DO_IDLE		0x0002
+#define BITDHT_QFLAGS_INTERNAL		0x0004  // runs through startup. (limited callback)
+#define BITDHT_QFLAGS_UPDATES		0x0008  // Do regular updates.
+
+/* Connect Callback Flags */
+#define BITDHT_CONNECT_CB_AUTH		1
+#define BITDHT_CONNECT_CB_PENDING	2
+#define BITDHT_CONNECT_CB_START		3
+#define BITDHT_CONNECT_CB_PROXY		4
+#define BITDHT_CONNECT_CB_FAILED	5
+#define BITDHT_CONNECT_CB_REQUEST	6
+
+#define BD_PROXY_CONNECTION_UNKNOWN_POINT       0
+#define BD_PROXY_CONNECTION_START_POINT         1
+#define BD_PROXY_CONNECTION_MID_POINT           2
+#define BD_PROXY_CONNECTION_END_POINT           3
+
+#define BITDHT_INFO_CB_TYPE_BADPEER	1
+
+/* Relay Modes */
+#define BITDHT_RELAYS_OFF		0
+#define BITDHT_RELAYS_ON		1
+#define BITDHT_RELAYS_ONLY		2
+#define BITDHT_RELAYS_SERVER		3
+
+
+class BitDhtCallback
+{
+	public:
+//        ~BitDhtCallback();
+
+	// dummy cos not needed for standard dht behaviour;
+	virtual int dhtNodeCallback(const bdId *  /*id*/, uint32_t /*peerflags*/)  { return 0; }
+
+	// must be implemented.
+	virtual int dhtPeerCallback(const bdId *id, uint32_t status) = 0;
+	virtual int dhtValueCallback(const bdNodeId *id, std::string key, uint32_t status) = 0;
+
+	// connection callback. Not required for basic behaviour, but forced for initial development.
+	virtual int dhtConnectCallback(const bdId *srcId, const bdId *proxyId, const bdId *destId,
+	uint32_t mode, uint32_t point, uint32_t param, uint32_t cbtype, uint32_t errcode) = 0; /*  { return 0; }  */
+
+	// Generic Info callback - initially will be used to provide bad peers.
+	virtual int dhtInfoCallback(const bdId *id, uint32_t type, uint32_t flags, std::string info) = 0;
+
+	// ask upper layer whether an IP is banned or not
+	// must not be implemented
+	// when set it will be used instead of the own ban list
+	// return code is used to express availability/absence
+	virtual int dhtIsBannedCallback(const sockaddr_in */*addr*/, bool */*isBanned*/) { return 0;}
+};
+
+
+class BitDhtInterface
+{
+	public:
+
+	/* bad peer notification */
+virtual void addBadPeer(const struct sockaddr_in &addr, uint32_t source, uint32_t reason, uint32_t age) = 0;
+
+	/* Friend Tracking */
+virtual void updateKnownPeer(const bdId *id, uint32_t type, uint32_t flags) = 0;
+
+        /***** Request Lookup (DHT Peer & Keyword) *****/
+virtual void addFindNode(bdNodeId *id, uint32_t mode) = 0;
+virtual void removeFindNode(bdNodeId *id) = 0;
+virtual void findDhtValue(bdNodeId *id, std::string key, uint32_t mode) = 0;
+
+	/***** Connections Requests *****/
+virtual bool ConnectionRequest(struct sockaddr_in *laddr, bdNodeId *target, uint32_t mode, uint32_t delay, uint32_t start) = 0;
+virtual void ConnectionAuth(bdId *srcId, bdId *proxyId, bdId *destId, uint32_t mode, uint32_t loc, 
+						uint32_t bandwidth, uint32_t delay, uint32_t answer) = 0;
+virtual void ConnectionOptions(uint32_t allowedModes, uint32_t flags) = 0;
+
+virtual bool setAttachMode(bool on) = 0;
+
+
+        /***** Add / Remove Callback Clients *****/
+virtual void addCallback(BitDhtCallback *cb) = 0;
+virtual void removeCallback(BitDhtCallback *cb) = 0;
+
+	/***** Get Results Details *****/
+virtual int getDhtPeerAddress(const bdNodeId *id, struct sockaddr_in &from) = 0;
+virtual int getDhtValue(const bdNodeId *id, std::string key, std::string &value) = 0;
+virtual int getDhtBucket(const int idx, bdBucket &bucket) = 0;
+
+virtual int getDhtQueries(std::map<bdNodeId, bdQueryStatus> &queries) = 0;
+virtual int getDhtQueryStatus(const bdNodeId *id, bdQuerySummary &query) = 0;
+
+        /* stats and Dht state */
+virtual int startDht() = 0;
+virtual int stopDht() = 0;
+virtual int stateDht() = 0; /* STOPPED, STARTING, ACTIVE, FAILED */
+virtual uint32_t statsNetworkSize() = 0;
+virtual uint32_t statsBDVersionSize() = 0; /* same version as us! */
+
+virtual uint32_t setDhtMode(uint32_t dhtFlags) = 0;
+};
+
+
+// general helper functions for decoding error messages.
+std::string decodeConnectionError(uint32_t errcode);
+std::string decodeConnectionErrorCRMove(uint32_t errcode);
+std::string decodeConnectionErrorSource(uint32_t errcode);
+std::string decodeConnectionErrorType(uint32_t errcode);
+
+
+#endif
+

libbitdht/src/bitdht/bdmanager.h

@@ -0,0 +1,209 @@
+/*******************************************************************************
+ * bitdht/bdmanager.h                                                          *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#ifndef BITDHT_MANAGER_H
+#define BITDHT_MANAGER_H
+
+/*******
+ * Node Manager.
+ ******/
+
+/******************************************
+ * 1) Maintains a list of ids to search for.
+ * 2) Sets up initial search for own node.
+ * 3) Checks on status of queries.
+ * 4) Callback on successful searches.
+ *
+ * This is pretty specific to RS requirements.
+ ****/
+
+#define BITDHT_PS_MASK_ACTIONS  	(0x000000ff)
+#define BITDHT_PS_MASK_STATE    	(0x0000ff00)
+
+#define BITDHT_PS_ACTION_SEARCHING 	(0x00000001)
+#define BITDHT_PS_ACTION_WAITING	(0x00000002)
+#define BITDHT_PS_ACTION_PINGING	(0x00000004)
+
+#define BITDHT_PS_STATE_UNKNOWN		(0x00000100)
+#define BITDHT_PS_STATE_OFFLINE		(0x00000200)
+#define BITDHT_PS_STATE_ONLINE		(0x00000400)
+#define BITDHT_PS_STATE_CONNECTED	(0x00000800)
+
+#include "bitdht/bdiface.h"
+#include "bitdht/bdnode.h"
+#include "util/bdbloom.h"
+
+
+
+class bdQueryPeer
+{
+	public:
+	bdId mId;
+	uint32_t mStatus;
+	uint32_t mQFlags;
+	//time_t mLastQuery;
+	//time_t mLastFound;
+	struct sockaddr_in mDhtAddr;
+	time_t mCallbackTS;   // for UPDATES flag.
+};
+
+
+#define BITDHT_MGR_STATE_OFF		0
+#define BITDHT_MGR_STATE_STARTUP	1
+#define BITDHT_MGR_STATE_FINDSELF	2
+#define BITDHT_MGR_STATE_ACTIVE 	3
+#define BITDHT_MGR_STATE_REFRESH 	4
+#define BITDHT_MGR_STATE_QUIET		5
+#define BITDHT_MGR_STATE_FAILED		6
+
+#define MAX_STARTUP_TIME 10
+#define MAX_REFRESH_TIME 10
+
+#define BITDHT_MGR_QUERY_FAILURE		1
+#define BITDHT_MGR_QUERY_PEER_OFFLINE		2
+#define BITDHT_MGR_QUERY_PEER_UNREACHABLE	3
+#define BITDHT_MGR_QUERY_PEER_ONLINE		4
+
+
+/*** NB: Nothing in here is protected by mutexes 
+ * must be done at a higher level!
+ ***/
+
+class bdNodeManager: public bdNode, public BitDhtInterface
+{
+	public:
+        bdNodeManager(bdNodeId *id, std::string dhtVersion, std::string bootfile, const std::string &filterfile, bdDhtFunctions *fns);
+
+
+void 	iteration();
+
+        /***** Functions to Call down to bdNodeManager ****/
+
+
+        /* Friend Tracking */
+virtual void addBadPeer(const struct sockaddr_in &addr, uint32_t source, uint32_t reason, uint32_t age);
+virtual void updateKnownPeer(const bdId *id, uint32_t type, uint32_t flags);
+
+        /* Request DHT Peer Lookup */
+        /* Request Keyword Lookup */
+virtual void addFindNode(bdNodeId *id, uint32_t mode);
+virtual void removeFindNode(bdNodeId *id);
+virtual void findDhtValue(bdNodeId *id, std::string key, uint32_t mode);
+
+        /***** Add / Remove Callback Clients *****/
+virtual void addCallback(BitDhtCallback *cb);
+virtual void removeCallback(BitDhtCallback *cb);
+
+        /***** Get Results Details *****/
+virtual int getDhtPeerAddress(const bdNodeId *id, struct sockaddr_in &from);
+virtual int getDhtValue(const bdNodeId *id, std::string key, std::string &value);
+virtual int getDhtBucket(const int idx, bdBucket &bucket);
+
+virtual int getDhtQueries(std::map<bdNodeId, bdQueryStatus> &queries);
+virtual int getDhtQueryStatus(const bdNodeId *id, bdQuerySummary &query);
+
+	/***** Connection Interface ****/
+virtual bool ConnectionRequest(struct sockaddr_in *laddr, bdNodeId *target, uint32_t mode, uint32_t delay, uint32_t start);
+virtual void ConnectionAuth(bdId *srcId, bdId *proxyId, bdId *destId, 
+						uint32_t mode, uint32_t loc, uint32_t bandwidth, uint32_t delay, uint32_t answer);
+virtual void ConnectionOptions(uint32_t allowedModes, uint32_t flags);
+
+virtual bool setAttachMode(bool on);
+
+	/* stats and Dht state */
+virtual int startDht();
+virtual int stopDht();
+virtual int stateDht(); /* STOPPED, STARTING, ACTIVE, FAILED */
+virtual uint32_t statsNetworkSize();
+virtual uint32_t statsBDVersionSize(); /* same version as us! */
+
+virtual uint32_t setDhtMode(uint32_t dhtFlags);
+
+        /******************* Internals *************************/
+
+	// Overloaded from bdnode for external node callback.
+virtual void addPeer(const bdId *id, uint32_t peerflags);
+        // Overloaded from bdnode for external node callback.
+virtual void callbackConnect(bdId *srcId, bdId *proxyId, bdId *destId,
+                                        int mode, int point, int param, int cbtype, int errcode); 
+
+int 	isBitDhtPacket(char *data, int size, struct sockaddr_in &from);
+
+	// this function is used by bdFilter (must be public!)
+void	doIsBannedCallback(const sockaddr_in *addr, bool *isAvailable, bool* isBanned);
+
+private:
+
+
+void 	doNodeCallback(const bdId *id, uint32_t peerflags);
+void 	doPeerCallback(const bdId *id, uint32_t status);
+void 	doValueCallback(const bdNodeId *id, std::string key, uint32_t status);
+void    doInfoCallback(const bdId *id, uint32_t type, uint32_t flags, std::string info);
+
+int	status();
+int	checkStatus();
+int 	checkPingStatus();
+int	checkBadPeerStatus();
+int 	SearchOutOfDate();
+void	startQueries();
+
+int     QueryRandomLocalNet();
+void    SearchForLocalNet();
+
+	std::map<bdNodeId, bdQueryPeer>	mActivePeers;
+        std::list<BitDhtCallback *> mCallbacks;
+
+	uint32_t mMode;
+	time_t   mModeTS;
+
+        time_t mStartTS;
+        time_t mSearchTS;
+        bool mSearchingDone;
+
+	bdDhtFunctions *mDhtFns;
+
+	uint32_t mNetworkSize;
+	uint32_t mBdNetworkSize;
+
+	bdBloom mBloomFilter;
+
+        bool mLocalNetEnhancements;
+
+	/* future node functions */
+	//addPeerPing(foundId);
+	//clearPing(it->first);
+	//PingStatus(it->first);
+};
+
+class bdDebugCallback: public BitDhtCallback
+{
+        public:
+        ~bdDebugCallback();
+virtual int dhtPeerCallback(const bdId *id, uint32_t status);
+virtual int dhtValueCallback(const bdNodeId *id, std::string key, uint32_t status);
+virtual int dhtConnectCallback(const bdId *srcId, const bdId *proxyId, const bdId *destId,             
+                         uint32_t mode, uint32_t point, uint32_t param, uint32_t cbtype, uint32_t errcode);
+virtual int dhtInfoCallback(const bdId *id, uint32_t type, uint32_t flags, std::string info);
+
+};
+
+
+#endif

libbitdht/src/bitdht/bdmsgs.h

@@ -0,0 +1,127 @@
+/*******************************************************************************
+ * bitdht/bdmsgs.h                                                             *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#ifndef BITDHT_MSGS_H
+#define BITDHT_MSGS_H
+
+#include <stdio.h>
+#include <inttypes.h>
+#include <list>
+#include "bitdht/bencode.h"
+#include "bitdht/bdobj.h"
+#include "bitdht/bdpeer.h"
+
+#define BITDHT_MSG_TYPE_UNKNOWN         0
+#define BITDHT_MSG_TYPE_PING            1
+#define BITDHT_MSG_TYPE_PONG            2
+#define BITDHT_MSG_TYPE_FIND_NODE       3
+#define BITDHT_MSG_TYPE_REPLY_NODE      4
+#define BITDHT_MSG_TYPE_GET_HASH        5
+#define BITDHT_MSG_TYPE_REPLY_HASH      6
+#define BITDHT_MSG_TYPE_REPLY_NEAR      7
+#define BITDHT_MSG_TYPE_POST_HASH       8
+#define BITDHT_MSG_TYPE_REPLY_POST      9
+
+
+
+// THESE ARE EXTENSIONS
+#define BITDHT_MSG_TYPE_CONNECT     	20
+
+// CONNECTIONS.
+#define BITDHT_MSG_TYPE_CONNECT_REQUEST     	101
+#define BITDHT_MSG_TYPE_CONNECT_REPLY     	102
+#define BITDHT_MSG_TYPE_CONNECT_START     	103
+#define BITDHT_MSG_TYPE_CONNECT_ACK     	104
+
+// FANCY HASHES.
+
+
+#define BITDHT_COMPACTNODEID_LEN 	26
+#define BITDHT_COMPACTPEERID_LEN 	6
+
+#define BE_Y_UNKNOWN    0
+#define BE_Y_R          1
+#define BE_Y_Q          2
+
+
+/****** Known BD Version Strings ******/
+
+#define BITDHT_VID_RS1	1
+#define BITDHT_VID_UT	2
+
+
+int bitdht_create_ping_msg(bdToken *tid, bdNodeId *id, bdToken *vid, char *msg, int avail);
+int bitdht_response_ping_msg(bdToken *tid, bdNodeId *id, bdToken *vid, char *msg, int avail); 
+int bitdht_find_node_msg(bdToken *tid, bdNodeId *id, bdNodeId *target, bool localnet, char *msg, int avail);
+int bitdht_resp_node_msg(bdToken *tid, bdNodeId *id, std::list<bdId> &nodes,
+                                        char *msg, int avail);
+int bitdht_get_peers_msg(bdToken *tid, bdNodeId *id, bdNodeId *info_hash,
+                                        char *msg, int avail);
+int bitdht_peers_reply_hash_msg(bdToken *tid, bdNodeId *id,
+                bdToken *token, std::list<std::string> &values,
+                                        char *msg, int avail);
+int bitdht_peers_reply_closest_msg(bdToken *tid, bdNodeId *id,
+                                bdToken *token, std::list<bdId> &nodes,
+                                        char *msg, int avail);
+int bitdht_announce_peers_msg(bdToken *tid, bdNodeId *id, bdNodeId *info_hash, 
+			uint32_t port, bdToken *token, char *msg, int avail);
+int bitdht_reply_announce_msg(bdToken *tid, bdNodeId *id,         
+                                        char *msg, int avail);
+
+
+// Extensions.
+int bitdht_connect_genmsg(bdToken *tid, bdNodeId *id, int msgtype, bdId *src, bdId *dest, int mode, int param, int status, char *msg, int avail);
+
+
+//int response_peers_message()
+//int response_closestnodes_message()
+
+be_node *beMsgGetDictNode(be_node *node, const char *key);
+int beMsgMatchString(be_node *n, const char *str, int len);
+uint32_t beMsgGetY(be_node *n);
+uint32_t beMsgType(be_node *n);
+
+bool bitdht_msgtype(uint32_t msg_type, std::string &name);
+
+
+uint32_t convertBdVersionToVID(bdVersion *version);
+
+be_node *makeCompactPeerIds(std::list<std::string> &values);
+be_node *makeCompactNodeIdString(std::list<bdId> &nodes);
+
+int beMsgGetToken(be_node *n, bdToken &token);
+int beMsgGetNodeId(be_node *n, bdNodeId &nodeId);
+int beMsgGetBdId(be_node *n, bdId &id);
+int beMsgGetListBdIds(be_node *n, std::list<bdId> &nodes);
+
+int beMsgGetListStrings(be_node *n, std::list<std::string> &values);
+int beMsgGetUInt32(be_node *n, uint32_t *port);
+
+/* Low Level conversion functions */
+int decodeCompactPeerId(struct sockaddr_in *addr, char *enc, int len);
+std::string encodeCompactPeerId(struct sockaddr_in *addr);
+
+int decodeCompactNodeId(bdId *id, char *enc, int len);
+std::string encodeCompactNodeId(bdId *id);
+
+
+
+#endif

libbitdht/src/bitdht/bdnode.h

@@ -0,0 +1,284 @@
+/*******************************************************************************
+ * bitdht/bdnode.h                                                             *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#ifndef BITDHT_NODE_H
+#define BITDHT_NODE_H
+
+#include "bitdht/bdpeer.h"
+#include "bitdht/bdquery.h"
+#include "bitdht/bdstore.h"
+#include "bitdht/bdobj.h"
+#include "bitdht/bdhash.h"
+#include "bitdht/bdhistory.h"
+#include "bitdht/bdfilter.h"
+
+#include "bitdht/bdconnection.h"
+#include "bitdht/bdaccount.h"
+
+#include "bitdht/bdfriendlist.h"
+
+class bdFilter;
+
+
+#define BD_QUERY_NEIGHBOURS		1
+#define BD_QUERY_LOCALNET		2
+#define BD_QUERY_HASH			3
+
+/**********************************
+ * Running a node....
+ *
+ * run().
+ * loops through and checks out of date peers.
+ * handles searches.
+ * prints out dht Table.
+ *
+
+The node handles the i/o traffic from peers.
+It 
+
+
+
+ping, return
+peers, return
+hash store, return
+hash get, return
+
+
+
+respond queue.
+
+query queue.
+
+
+
+
+
+input -> call into recvFunction()
+output -> call back to Udp().
+
+
+
+
+ *********/
+
+class bdFilteredPeer ;
+class bdNodeManager;
+
+class bdNodeNetMsg
+{
+
+	public:
+	bdNodeNetMsg(char *data, int size, struct sockaddr_in *addr);
+	~bdNodeNetMsg();
+
+	void print(std::ostream &out);
+
+	char *data;
+	int mSize;
+	struct sockaddr_in addr;
+
+};
+
+class bdNodePublisher
+{
+	public:
+	/* simplified outgoing msg functions (for the managers) */
+	virtual void send_ping(bdId *id) = 0; /* message out */
+	virtual void send_query(bdId *id, bdNodeId *targetNodeId, bool localnet) = 0; /* message out */
+	virtual void send_connect_msg(bdId *id, int msgtype, 
+				bdId *srcAddr, bdId *destAddr, int mode, int param, int status) = 0;
+
+        // internal Callback -> normally continues to callbackConnect().
+        virtual void callbackConnect(bdId *srcId, bdId *proxyId, bdId *destId,
+                                int mode, int point, int param, int cbtype, int errcode) = 0;
+
+};
+
+
+class bdNode: public bdNodePublisher
+{
+	public:
+
+    bdNode(bdNodeId *id, std::string dhtVersion, const std::string& bootfile, const std::string& filterfile,
+		bdDhtFunctions *fns, bdNodeManager* manager);
+
+	void init(); /* sets up the self referential classes (mQueryMgr & mConnMgr) */
+
+	void setNodeOptions(uint32_t optFlags);
+	uint32_t setNodeDhtMode(uint32_t dhtFlags);
+
+	/* startup / shutdown node */
+	void restartNode();
+	void shutdownNode();
+
+	void getOwnId(bdNodeId *id);
+
+	// virtual so manager can do callback.
+	// peer flags defined in bdiface.h
+	virtual void addPeer(const bdId *id, uint32_t peerflags);
+
+	void printState();
+	void checkPotentialPeer(bdId *id, bdId *src);
+	void addPotentialPeer(bdId *id, bdId *src);
+
+	void iterationOff();
+	void iteration();
+	void processRemoteQuery();
+	void updateStore();
+
+    bool addressBanned(const sockaddr_in &raddr) ;
+    bool getFilteredPeers(std::list<bdFilteredPeer> &peers);
+    //void loadFilteredPeers(const std::list<bdFilteredPeer> &peers);
+
+	/* simplified outgoing msg functions (for the managers) */
+	virtual void send_ping(bdId *id); /* message out */
+	virtual void send_query(bdId *id, bdNodeId *targetNodeId, bool localnet); /* message out */
+	virtual void send_connect_msg(bdId *id, int msgtype, 
+				bdId *srcAddr, bdId *destAddr, int mode, int param, int status);
+
+// This is implemented in bdManager.
+//        virtual void callbackConnect(bdId *srcId, bdId *proxyId, bdId *destId,
+//                                int mode, int point, int param, int cbtype, int errcode);
+
+	/* interaction with outside world (Accessed by controller to deliver us msgs) */
+int 	outgoingMsg(struct sockaddr_in *addr, char *msg, int *len);
+void 	incomingMsg(struct sockaddr_in *addr, char *msg, int len);
+
+	// For Relay Mode switching.
+void	dropRelayServers();
+void	pingRelayServers();
+
+// Below is internal Management of incoming / outgoing messages.
+
+private:
+
+	/* internal interaction with network */
+void	sendPkt(char *msg, int len, struct sockaddr_in addr);
+void	recvPkt(char *msg, int len, struct sockaddr_in addr);
+
+
+	/* output functions (send msg) */
+	void msgout_ping(bdId *id, bdToken *transId);
+	void msgout_pong(bdId *id, bdToken *transId);
+	void msgout_find_node(bdId *id, bdToken *transId, bdNodeId *query, bool localnet);
+	void msgout_reply_find_node(bdId *id, bdToken *transId, 
+						std::list<bdId> &peers);
+	void msgout_get_hash(bdId *id, bdToken *transId, bdNodeId *info_hash);
+	void msgout_reply_hash(bdId *id, bdToken *transId, 
+				bdToken *token, std::list<std::string> &values);
+	void msgout_reply_nearest(bdId *id, bdToken *transId, 
+				bdToken *token, std::list<bdId> &peers);
+
+	void msgout_post_hash(bdId *id, bdToken *transId, bdNodeId *info_hash, 
+				uint32_t port, bdToken *token);
+	void msgout_reply_post(bdId *id, bdToken *transId);
+
+
+	/* input functions (once mesg is parsed) */
+	uint32_t parseVersion(bdToken *versionId);
+	void msgin_ping(bdId *id, bdToken *token, bdToken *versionId);
+	void msgin_pong(bdId *id, bdToken *transId, bdToken *versionId);
+
+	void msgin_find_node(bdId *id, bdToken *transId, bdNodeId *query, bool localnet);
+	void msgin_reply_find_node(bdId *id, bdToken *transId, 
+						std::list<bdId> &entries);
+
+	void msgin_get_hash(bdId *id, bdToken *transId, bdNodeId *nodeid);
+	void msgin_reply_hash(bdId *id, bdToken *transId, 
+				bdToken *token, std::list<std::string> &values);
+	void msgin_reply_nearest(bdId *id, bdToken *transId, 
+				bdToken *token, std::list<bdId> &nodes);
+
+	void msgin_post_hash(bdId *id,  bdToken *transId,  
+				bdNodeId *info_hash,  uint32_t port, bdToken *token);
+	void msgin_reply_post(bdId *id, bdToken *transId);
+
+	void msgout_connect_genmsg(bdId *id, bdToken *transId, int msgtype, 
+				bdId *srcAddr, bdId *destAddr, int mode, int param, int status);
+	void msgin_connect_genmsg(bdId *id, bdToken *transId, int msgtype,
+                                        bdId *srcAddr, bdId *destAddr, int mode, int param, int status);
+
+
+
+	/* token handling */
+	void genNewToken(bdToken *token);
+	int queueQuery(bdId *id, bdNodeId *query, bdToken *transId, uint32_t query_type);
+
+	/* transId handling */
+	void genNewTransId(bdToken *token);
+	void registerOutgoingMsg(bdId *id, bdToken *transId, uint32_t msgType, bdNodeId *aboutId);
+	uint32_t registerIncomingMsg(bdId *id, bdToken *transId, uint32_t msgType, bdNodeId *aboutId);
+
+	void cleanupTransIdRegister();
+
+
+	void doStats();
+
+	/********** Variables **********/
+	private:
+
+	/**** Some Variables are Protected to allow inherited classes to use *****/
+	protected:
+
+	bdSpace mNodeSpace;
+	bdFilter mFilterPeers;
+
+	bdQueryManager *mQueryMgr;
+	bdConnectManager *mConnMgr;
+
+	bdNodeId mOwnId;
+	bdId 	mLikelyOwnId; // Try to workout own id address.
+	std::string mDhtVersion;
+
+	bdAccount mAccount;
+	bdStore mStore;
+
+	bdDhtFunctions *mFns;
+	bdHashSpace mHashSpace;
+
+	bdFriendList mFriendList;
+	bdPeerQueue  mBadPeerQueue;
+
+	bdHistory mHistory; /* for understanding the DHT */
+
+	bdQueryHistory mQueryHistory; /* for determining old peers */
+
+	private:
+
+	uint32_t mNodeOptionFlags;	
+	uint32_t mNodeDhtMode;
+
+	uint32_t mMaxAllowedMsgs;
+	uint32_t mRelayMode;
+
+
+	std::list<bdRemoteQuery> mRemoteQueries;
+
+	std::list<bdId> mPotentialPeers;
+
+	std::list<bdNodeNetMsg *> mOutgoingMsgs;
+	std::list<bdNodeNetMsg *> mIncomingMsgs;
+
+};
+
+
+
+#endif // BITDHT_NODE_H

libbitdht/src/bitdht/bdobj.cc

@@ -0,0 +1,48 @@
+/*******************************************************************************
+ * bitdht/bdobj.cc                                                             *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "bitdht/bdobj.h"
+
+
+void bdPrintTransId(std::ostream &out, bdToken *transId)
+{
+	//out << transId->data;
+	bdPrintToken(out, transId);
+	return;
+}
+
+
+void bdPrintToken(std::ostream &out, bdToken *token)
+{
+	for(unsigned int i = 0; i < token->len; i++)
+	{
+		out << std::hex << (uint32_t) token->data[i];
+	}
+	out << std::dec;
+}
+
+void bdPrintCompactPeerId(std::ostream &out, std::string /*cpi*/ )
+{
+	out << "DummyCompactPeerId";
+}
+
+

libbitdht/src/bitdht/bdobj.h

@@ -0,0 +1,59 @@
+/*******************************************************************************
+ * bitdht/bdobj.h                                                              *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#ifndef BITDHT_OBJECTS_H
+#define BITDHT_OBJECTS_H
+
+#define BITDHT_TOKEN_MAX_LEN 20
+
+#include <iostream>
+#include <inttypes.h>
+
+class bdToken
+{
+	public:
+	bdToken() :len(0) { return; }
+	uint32_t len;
+	unsigned char data[BITDHT_TOKEN_MAX_LEN];
+};
+
+class bdCompactIds
+{
+	public:
+	bdCompactIds() :len(0) { return; }
+	uint32_t len;
+	unsigned char data[BITDHT_TOKEN_MAX_LEN];
+};
+
+class bdVersion
+{
+	public:
+	bdVersion() :len(0) { return; }
+	uint32_t len;
+	unsigned char data[BITDHT_TOKEN_MAX_LEN];
+};
+
+void bdPrintTransId(std::ostream &out, bdToken *transId);
+void bdPrintToken(std::ostream &out, bdToken *transId);
+void bdPrintCompactPeerId(std::ostream &out, std::string cpi);
+
+#endif
+

libbitdht/src/bitdht/bdpeer.h

@@ -0,0 +1,212 @@
+/*******************************************************************************
+ * bitdht/bdpeer.h                                                             *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#ifndef BITDHT_PEER_H
+#define BITDHT_PEER_H
+
+#include "bitdht/bdiface.h"
+#include <time.h>
+
+/*******
+ * These type of parameters are now DHT Function dependent
+ *
+#define BITDHT_BUCKET_SIZE 20
+#define BITDHT_BUCKET_SIZE_BITS 5
+
+#define BITDHT_N_BUCKETS  BITDHT_KEY_BITLEN
+ *
+ *
+ ***/
+
+/*** 
+ * DEFINED in bdiface.h
+ * #define BITDHT_KEY_LEN 20
+ * #define BITDHT_KEY_INTLEN 5
+ * #define BITDHT_KEY_BITLEN 160
+ ***/
+
+#define BITDHT_ULLONG_BITS 64
+
+#define BITDHT_MAX_RESPONSE_PERIOD      (15)
+#define BITDHT_MAX_SEND_PERIOD	300   // 5 minutes.
+#define BITDHT_MAX_RECV_PERIOD	(BITDHT_MAX_SEND_PERIOD + BITDHT_MAX_RESPONSE_PERIOD) // didn't respond to a ping.
+
+// Properly out of date.
+#define BITDHT_DISCARD_PERIOD		(2 * BITDHT_MAX_SEND_PERIOD + BITDHT_MAX_RESPONSE_PERIOD) // didn't respond to two pings.
+
+// Must have a FLAG by this time. (Make it really quick - so we through away the rubbish).
+
+
+#include <list>
+#include <string>
+#include <map>
+#include <vector>
+
+/****
+ * DEFINED in bdiface.h
+ *
+ * class bdNodeId
+ * {
+ * 	public:
+ *	unsigned char data[BITDHT_KEY_LEN];
+ * };
+ ****/
+
+
+/****
+ * DEFINED in bdiface.h
+ *
+
+class bdMetric: public bdNodeId {};
+
+class bdId
+{
+	public:
+
+	bdId();
+	bdId(bdNodeId in_id, struct sockaddr_in in_addr);
+
+	struct sockaddr_in addr;
+	bdNodeId id;
+};
+ *
+ *********/
+
+
+
+//void bdRandomNodeId(bdNodeId *id);
+
+// Only Functions that are common for all Dhts.
+// zero, basic comparisons..
+void bdZeroNodeId(bdNodeId *id);
+
+//void bdRandomId(bdId *id);
+//int bdDistance(const bdNodeId *a, const bdNodeId *b, bdMetric *r);
+//int bdBucketDistance(const bdMetric *m);
+//int bdBucketDistance(const bdNodeId *a, const bdNodeId *b);
+//int operator<(const bdMetric &a, const bdMetric &b);
+
+//int operator<(const struct sockaddr_in &a, const struct sockaddr_in &b);
+
+int operator<(const bdNodeId &a, const bdNodeId &b);
+int operator<(const bdId &a, const bdId &b);
+int operator==(const bdNodeId &a, const bdNodeId &b);
+int operator==(const bdId &a, const bdId &b);
+
+//void bdRandomMidId(const bdNodeId *target, const bdNodeId *other, bdNodeId *mid);
+
+//void bdPrintId(std::ostream &out, const bdId *a);
+//void bdPrintNodeId(std::ostream &out, const bdNodeId *a);
+
+//std::string bdConvertToPrintable(std::string input);
+
+/****
+ * DEFINED in bdiface.h
+ *
+
+class bdPeer
+{
+	public:
+
+	bdId   mPeerId;
+	uint32_t mPeerFlags;
+	time_t mLastSendTime;
+	time_t mLastRecvTime;
+	time_t mFoundTime;     // time stamp that peer was found 
+};
+
+
+class bdBucket
+{
+	public:
+
+	bdBucket();
+
+	// list so we can queue properly 
+	std::list<bdPeer> entries;
+};
+ *
+ *
+ *****/
+
+class bdSpace
+{
+	public:
+
+	bdSpace(bdNodeId *ownId, bdDhtFunctions *fns);
+
+int	clear();
+
+int	setAttachedFlag(uint32_t withflags, int count);
+
+	/* accessors */
+int 	find_nearest_nodes(const bdNodeId *id, int number, 
+		std::multimap<bdMetric, bdId> &nearest);
+
+int 	find_nearest_nodes_with_flags(const bdNodeId *id, int number, 
+		std::list<bdId> excluding, 
+		std::multimap<bdMetric, bdId> &nearest, uint32_t with_flag);
+
+int 	find_node(const bdNodeId *id, int number, 
+		std::list<bdId> &matchIds, uint32_t with_flag);
+int 	find_exactnode(const bdId *id, bdPeer &peer);
+
+// switched to more efficient single sweep.
+//int	out_of_date_peer(bdId &id); // side-effect updates, send flag on peer.
+int     scanOutOfDatePeers(std::list<bdId> &peerIds);
+int     updateAttachedPeers();
+
+int     add_peer(const bdId *id, uint32_t mode);
+int     printDHT();
+int     getDhtBucket(const int idx, bdBucket &bucket);
+
+uint32_t calcNetworkSize();
+uint32_t calcNetworkSizeWithFlag(uint32_t withFlag);
+uint32_t calcNetworkSizeWithFlag_old(uint32_t withFlag);
+uint32_t calcSpaceSize();
+uint32_t calcSpaceSizeWithFlag(uint32_t withFlag);
+
+        /* special function to enable DHT localisation (i.e find peers from own network) */
+bool 	findRandomPeerWithFlag(bdId &id, uint32_t withFlag);
+
+	/* strip out flags - to switch in/out of relay mode */
+int 	clean_node_flags(uint32_t flags);
+
+	/* to add later */
+int	updateOwnId(bdNodeId *newOwnId);
+
+	/* flag peer */
+bool	flagpeer(const bdId *id, uint32_t flags, uint32_t ex_flags);
+
+	private:
+
+	std::vector<bdBucket> buckets;
+	bdNodeId mOwnId;
+	bdDhtFunctions *mFns;
+
+	uint32_t mAttachedFlags;
+	uint32_t mAttachedCount;
+	time_t   mAttachTS;
+};
+
+
+#endif
+

libbitdht/src/bitdht/bdquery.h

@@ -0,0 +1,157 @@
+/*******************************************************************************
+ * bitdht/bdquery.h                                                            *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2010 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#ifndef BITDHT_QUERY_H
+#define BITDHT_QUERY_H
+
+#include "bitdht/bdiface.h"
+#include "bitdht/bdpeer.h"
+#include "bitdht/bdobj.h"
+
+/* Query result flags are in bdiface.h */
+
+#define BITDHT_MIN_QUERY_AGE		10
+#define BITDHT_MAX_QUERY_AGE		300 /* Query Should take <1 minute, so 5 minutes sounds reasonable */
+
+class bdQuery
+{
+	public:
+	bdQuery(const bdNodeId *id, std::list<bdId> &startList, uint32_t queryFlags, 
+		bdDhtFunctions *fns);
+
+	// get the answer.
+bool	result(std::list<bdId> &answer);
+bool	proxies(std::list<bdId> &answer);
+bool	potentialProxies(std::list<bdId> &answer);
+
+	// returning results get passed to all queries.
+//void 	addNode(const bdId *id, int mode);		
+int 	nextQuery(bdId &id, bdNodeId &targetId);
+int 	addPeer(const bdId *id, uint32_t mode);
+int 	addPotentialPeer(const bdId *id, const bdId *src, uint32_t srcmode);
+int 	printQuery();
+
+	// searching for
+	bdNodeId mId;
+	bdMetric mLimit;
+	uint32_t mState;
+	time_t mQueryTS;
+	uint32_t mQueryFlags;
+	int32_t mSearchTime;
+
+	int32_t mQueryIdlePeerRetryPeriod; // seconds between retries.
+
+	//private:
+
+	// Closest Handling Fns.
+int 	addClosestPeer(const bdId *id, uint32_t mode);
+
+	// Potential Handling Fns.
+int 	worthyPotentialPeer(const bdId *id);
+int 	updatePotentialPeer(const bdId *id, uint32_t mode, uint32_t addType);
+int 	trimPotentialPeers_FixedLength();
+int 	trimPotentialPeers_toClosest();
+int 	removeOldPotentialPeers();
+
+	// Proxy Handling Fns.
+int 	addProxy(const bdId *id, const bdId *src, uint32_t srcmode);
+int 	updateProxy(const bdId *id, uint32_t mode);
+int 	updateProxyList(const bdId *id, uint32_t mode, std::list<bdPeer> &searchProxyList);
+int 	trimProxies();
+
+
+	// closest peers.
+	std::multimap<bdMetric, bdPeer>  mClosest;
+	std::multimap<bdMetric, bdPeer>  mPotentialPeers;
+	time_t mPotPeerCleanTS; // periodic cleanup of PotentialPeers.
+
+	uint32_t mRequiredPeerFlags; 
+	std::list<bdPeer>  mProxiesUnknown;
+	std::list<bdPeer>  mProxiesFlagged;
+
+	int mClosestListSize;
+	bdDhtFunctions *mFns;
+
+};
+
+#if 0
+
+class bdQueryStatus
+{
+	public:
+	uint32_t mStatus;
+	uint32_t mQFlags;
+	std::list<bdId> mResults;
+};
+
+#endif
+
+
+
+/* this is just a container class.
+ * we locally seach for this, once then discard.
+ */
+class bdRemoteQuery
+{
+	public:
+	bdRemoteQuery(bdId *id, bdNodeId *query, bdToken *transId, uint32_t query_type);
+
+	bdId mId;
+	bdNodeId mQuery;
+	bdToken mTransId;
+	uint32_t mQueryType;
+
+	time_t mQueryTS;
+};
+
+
+
+class bdQueryHistoryList
+{
+	public:
+	bdQueryHistoryList();
+
+bool    addIncomingQuery(time_t recvd, const bdNodeId *aboutId); // calcs and returns mBadPeer
+bool	cleanupMsgs(time_t before);				 // returns true if empty.
+
+	bool mBadPeer;
+	std::multimap<time_t, bdNodeId> mList;
+};
+
+class bdQueryHistory
+{
+	public:
+	bdQueryHistory();
+
+bool    addIncomingQuery(time_t recvd, const bdId *id, const bdNodeId *aboutId);
+void    printMsgs();
+
+void    cleanupOldMsgs();
+
+bool    isBadPeer(const bdId *id);
+
+	int mStorePeriod;
+        std::map<bdId, bdQueryHistoryList> mHistory;
+};
+
+
+#endif
+

libbitdht/src/bitdht/bdquerymgr.cc

@@ -0,0 +1,379 @@
+/*******************************************************************************
+ * bitdht/bdquerymgr.cc                                                        *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2011 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "bitdht/bdquerymgr.h"
+#include "bitdht/bdnode.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <iostream>
+#include <iomanip>
+
+
+#define BITDHT_QUERY_START_PEERS    10
+#define BITDHT_QUERY_NEIGHBOUR_PEERS    8
+#define BITDHT_MAX_REMOTE_QUERY_AGE	10
+
+/****
+ * #define DEBUG_NODE_MULTIPEER 1
+ * #define DEBUG_NODE_MSGS 1
+ * #define DEBUG_NODE_ACTIONS 1
+
+ * #define DEBUG_NODE_MSGIN 1
+ * #define DEBUG_NODE_MSGOUT 1
+ ***/
+
+//#define DEBUG_NODE_MSGS 1
+
+
+bdQueryManager::bdQueryManager(bdSpace *space, bdDhtFunctions *fns, bdNodePublisher *pub)
+	:mNodeSpace(space), mFns(fns), mPub(pub)
+{
+}
+
+/***** Startup / Shutdown ******/
+void bdQueryManager::shutdownQueries()
+{
+	/* clear the queries */
+	std::list<bdQuery *>::iterator it;
+	for(it = mLocalQueries.begin(); it != mLocalQueries.end();it++)
+	{
+		delete (*it);
+	}
+
+	mLocalQueries.clear();
+}
+
+
+void bdQueryManager::printQueries()
+{
+	std::cerr << "bdQueryManager::printQueries()";
+	std::cerr << std::endl;
+
+	int i = 0;
+	std::list<bdQuery *>::iterator it;
+	for(it = mLocalQueries.begin(); it != mLocalQueries.end(); it++, i++)
+	{
+		fprintf(stderr, "Query #%d:\n", i);
+		(*it)->printQuery();
+		fprintf(stderr, "\n");
+	}
+}
+
+
+int bdQueryManager::iterateQueries(int maxQueries)
+{
+#ifdef DEBUG_NODE_MULTIPEER 
+	std::cerr << "bdQueryManager::iterateQueries() of Peer: ";
+	mFns->bdPrintNodeId(std::cerr, &mOwnId);
+	std::cerr << std::endl;
+#endif
+
+	/* allow each query to send up to one query... until maxMsgs has been reached */
+	int numQueries = mLocalQueries.size();
+	int sentQueries = 0;
+	int i = 0;
+	
+	bdId id;
+	bdNodeId targetNodeId;
+	
+	while((i < numQueries) && (sentQueries < maxQueries))
+	{
+		bdQuery *query = mLocalQueries.front();
+		mLocalQueries.pop_front();
+		mLocalQueries.push_back(query);
+
+		/* go through the possible queries */
+		if (query->nextQuery(id, targetNodeId))
+		{
+#ifdef DEBUG_NODE_MSGS 
+			std::cerr << "bdQueryManager::iteration() send_query(";
+			mFns->bdPrintId(std::cerr, &id);
+			std::cerr << ",";
+			mFns->bdPrintNodeId(std::cerr, &targetNodeId);
+			std::cerr << ")";
+			std::cerr << std::endl;
+#endif
+			mPub->send_query(&id, &targetNodeId, false);
+			sentQueries++;
+		}
+		i++;
+	}
+	
+#ifdef DEBUG_NODE_ACTIONS 
+	std::cerr << "bdQueryManager::iteration() maxMsgs: " << maxMsgs << " sentPings: " << sentPings;
+	std::cerr << " / " << allowedPings;
+	std::cerr << " sentQueries: " << sentQueries;
+	std::cerr << " / " << numQueries;
+	std::cerr << std::endl;
+#endif
+
+	//printQueries();
+
+	return sentQueries;
+}
+
+
+bool bdQueryManager::checkPotentialPeer(bdId *id, bdId *src)
+{
+	bool isWorthyPeer = false;
+	/* also push to queries */
+	std::list<bdQuery *>::iterator it;
+	for(it = mLocalQueries.begin(); it != mLocalQueries.end(); it++)
+	{
+		if ((*it)->addPotentialPeer(id, src, 0))
+		{
+			isWorthyPeer = true;
+		}
+	}
+
+	if (!isWorthyPeer)
+	{
+		isWorthyPeer = checkWorthyPeerSources(src);
+	}
+
+	return isWorthyPeer;
+}
+
+
+void bdQueryManager::addPeer(const bdId *id, uint32_t peerflags)
+{
+
+#ifdef DEBUG_NODE_ACTIONS 
+	fprintf(stderr, "bdQueryManager::addPeer(");
+	mFns->bdPrintId(std::cerr, id);
+	fprintf(stderr, ")\n");
+#endif
+
+	/* iterate through queries */
+	std::list<bdQuery *>::iterator it;
+	for(it = mLocalQueries.begin(); it != mLocalQueries.end(); it++)
+	{
+		(*it)->addPeer(id, peerflags);
+	}
+}
+
+
+/************************************ Query Details        *************************/
+void bdQueryManager::addQuery(const bdNodeId *id, uint32_t qflags)
+{
+
+	std::list<bdId> startList;
+	std::multimap<bdMetric, bdId> nearest;
+	std::multimap<bdMetric, bdId>::iterator it;
+
+	mNodeSpace->find_nearest_nodes(id, BITDHT_QUERY_START_PEERS, nearest);
+
+#ifdef DEBUG_NODE_ACTIONS 
+	fprintf(stderr, "bdQueryManager::addQuery(");
+	mFns->bdPrintNodeId(std::cerr, id);
+	fprintf(stderr, ")\n");
+#endif
+
+	for(it = nearest.begin(); it != nearest.end(); it++)
+	{
+		startList.push_back(it->second);
+	}
+
+	bdQuery *query = new bdQuery(id, startList, qflags, mFns);
+	mLocalQueries.push_back(query);
+}
+
+
+void bdQueryManager::clearQuery(const bdNodeId *rmId)
+{
+	std::list<bdQuery *>::iterator it;
+	for(it = mLocalQueries.begin(); it != mLocalQueries.end();)
+	{
+		if ((*it)->mId == *rmId)
+		{
+			bdQuery *query = (*it);
+			it = mLocalQueries.erase(it);
+			delete query;
+		}
+		else
+		{
+			it++;
+		}
+	}
+}
+
+void bdQueryManager::QueryStatus(std::map<bdNodeId, bdQueryStatus> &statusMap)
+{
+	std::list<bdQuery *>::iterator it;
+	for(it = mLocalQueries.begin(); it != mLocalQueries.end(); it++)
+	{
+		bdQueryStatus status;
+		status.mStatus = (*it)->mState;
+		status.mQFlags = (*it)->mQueryFlags;
+		(*it)->result(status.mResults);
+		statusMap[(*it)->mId] = status;
+	}
+}
+
+int bdQueryManager::QuerySummary(const bdNodeId *id, bdQuerySummary &query)
+{
+	std::list<bdQuery *>::iterator it;
+	for(it = mLocalQueries.begin(); it != mLocalQueries.end(); it++)
+	{
+		if ((*it)->mId == *id)
+		{
+			query.mId = (*it)->mId;
+			query.mLimit = (*it)->mLimit;
+			query.mState = (*it)->mState;
+			query.mQueryTS = (*it)->mQueryTS;
+			query.mQueryFlags = (*it)->mQueryFlags;
+			query.mSearchTime = (*it)->mSearchTime;
+			query.mClosest = (*it)->mClosest;
+			query.mPotentialPeers = (*it)->mPotentialPeers;
+			query.mProxiesUnknown = (*it)->mProxiesUnknown;
+			query.mProxiesFlagged = (*it)->mProxiesFlagged;
+ 			query.mQueryIdlePeerRetryPeriod = (*it)->mQueryIdlePeerRetryPeriod;
+
+			return 1;
+		}
+	}
+	return 0;
+}
+
+/* Extract Results from Peer Queries */
+
+#define BDQRYMGR_RESULTS 	1
+#define BDQRYMGR_PROXIES	2
+#define BDQRYMGR_POTPROXIES	3
+
+int  bdQueryManager::getResults(bdNodeId *target, std::list<bdId> &answer, int querytype)
+{
+		
+	/* grab any peers from any existing query */
+	int results = 0;
+	std::list<bdQuery *>::iterator qit;
+	for(qit = mLocalQueries.begin(); qit != mLocalQueries.end(); qit++)
+	{
+		if (!((*qit)->mId == (*target)))
+		{
+			continue;
+		}
+		
+#ifdef DEBUG_NODE_CONNECTION
+		std::cerr << "bdQueryManager::getResults() Found Matching Query";
+		std::cerr << std::endl;
+#endif
+		switch(querytype)
+		{
+			default:
+			case BDQRYMGR_RESULTS:
+				results = (*qit)->result(answer);
+				break;
+			case BDQRYMGR_PROXIES:
+				results = (*qit)->proxies(answer);
+				break;
+			case BDQRYMGR_POTPROXIES:
+				results = (*qit)->potentialProxies(answer);
+				break;
+		}
+		/* will only be one matching query.. so end loop */
+		return results;
+	}
+	return 0;
+}
+
+		
+int  bdQueryManager::result(bdNodeId *target, std::list<bdId> &answer)
+{
+	return getResults(target, answer,  BDQRYMGR_RESULTS);
+}
+		
+int  bdQueryManager::proxies(bdNodeId *target, std::list<bdId> &answer)
+{
+	return getResults(target, answer,  BDQRYMGR_PROXIES);
+}
+		
+int  bdQueryManager::potentialProxies(bdNodeId *target, std::list<bdId> &answer)
+{
+	return getResults(target, answer,  BDQRYMGR_POTPROXIES);
+}
+
+
+/************ WORTHY PEERS **********/
+
+#define MAX_WORTHY_PEER_AGE 15
+
+void bdQueryManager::addWorthyPeerSource(bdId *src)
+{
+	time_t now = time(NULL);
+
+	bdPeer peer;
+	peer.mPeerId = *src;
+	peer.mFoundTime = now;
+
+#ifdef DEBUG_NODE_ACTIONS 
+	std::cerr << "bdQueryManager::addWorthyPeerSource(";
+	mFns->bdPrintId(std::cerr, src);
+	std::cerr << ")" << std::endl;
+#endif
+
+	mWorthyPeerSources.push_back(peer);
+}
+
+bool bdQueryManager::checkWorthyPeerSources(bdId *src)
+{
+	if (!src)
+		return false;
+
+	time_t now = time(NULL);
+	std::list<bdPeer>::iterator it;
+	for(it = mWorthyPeerSources.begin(); it != mWorthyPeerSources.end(); )
+	{
+		if (now - it->mFoundTime > MAX_WORTHY_PEER_AGE)
+		{
+#ifdef DEBUG_NODE_ACTIONS 
+			std::cerr << "bdQueryManager::checkWorthyPeerSource() Discard old Source: ";
+			mFns->bdPrintId(std::cerr, &(it->mPeerId));
+			std::cerr << std::endl;
+#endif
+
+			it = mWorthyPeerSources.erase(it);
+		}
+		else
+		{
+			if (it->mPeerId == *src)
+			{
+#ifdef DEBUG_NODE_ACTIONS 
+				std::cerr << "bdQueryManager::checkWorthyPeerSource(";
+				mFns->bdPrintId(std::cerr, src);
+				std::cerr << ") = true" << std::endl;
+#endif
+
+				return true;
+			}
+			it++;
+		}
+	}
+	return false;
+}
+
+
+

libbitdht/src/bitdht/bdquerymgr.h

@@ -0,0 +1,73 @@
+/*******************************************************************************
+ * bitdht/bdquerymgr.h                                                         *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2011 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#ifndef BITDHT_QUERY_MANAGER_H
+#define BITDHT_QUERY_MANAGER_H
+
+#include "bitdht/bdquery.h"
+class bdNodePublisher;
+
+
+class bdQueryManager
+{
+	public:
+
+	bdQueryManager(bdSpace *space, bdDhtFunctions *fns, bdNodePublisher *pub);	
+
+	void shutdownQueries();
+	void printQueries();
+
+	int iterateQueries(int maxqueries);
+
+	bool checkPotentialPeer(bdId *id, bdId *src);
+	void addPeer(const bdId *id, uint32_t peerflags);
+
+	void addQuery(const bdNodeId *id, uint32_t qflags);
+	void clearQuery(const bdNodeId *id);
+	void QueryStatus(std::map<bdNodeId, bdQueryStatus> &statusMap);
+	int  QuerySummary(const bdNodeId *id, bdQuerySummary &query);
+
+	int  result(bdNodeId *target, std::list<bdId> &answer);
+	int  proxies(bdNodeId *target, std::list<bdId> &answer);
+	int  potentialProxies(bdNodeId *target, std::list<bdId> &answer);
+
+	// extra "Worthy Peers" we will want to ping.
+	void addWorthyPeerSource(bdId *src);
+	bool checkWorthyPeerSources(bdId *src);
+
+	private:
+
+	int getResults(bdNodeId *target, std::list<bdId> &answer, int querytype);
+
+	/* NB: No Mutex Protection... Single threaded, Mutex at higher level!
+	 */
+
+	bdSpace *mNodeSpace;
+	bdDhtFunctions *mFns;
+	bdNodePublisher *mPub;
+
+	std::list<bdQuery *> mLocalQueries;
+	std::list<bdPeer> mWorthyPeerSources;
+};
+
+
+
+#endif // BITDHT_QUERY_MANAGER_H

libbitdht/src/bitdht/bdstddht.cc

@@ -0,0 +1,347 @@
+/*******************************************************************************
+ * bitdht/bdstddht.cc                                                          *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2011 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "bitdht/bdstddht.h"
+#include "bitdht/bdpeer.h"
+#include "util/bdrandom.h"
+#include "util/bdstring.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <limits.h>
+
+#include <iostream>
+#include <iomanip>
+
+/**
+ * #define BITDHT_DEBUG 1
+**/
+
+void bdStdRandomId(bdId *id)
+{
+	bdStdRandomNodeId(&(id->id));
+	id->addr.sin_addr.s_addr = bdRandom::random_u32();
+	id->addr.sin_port = (bdRandom::random_u32() % USHRT_MAX);
+
+	return;
+}
+
+void bdStdRandomNodeId(bdNodeId *id)
+{
+	uint32_t *a_data = (uint32_t *) id->data;
+	for(int i = 0; i < BITDHT_KEY_INTLEN; i++)
+	{
+		a_data[i] = bdRandom::random_u32();
+	}
+	return;
+}
+
+void bdStdZeroNodeId(bdNodeId *id)
+{
+	uint32_t *a_data = (uint32_t *) id->data;
+	for(int i = 0; i < BITDHT_KEY_INTLEN; i++)
+	{
+		a_data[i] = 0;
+	}
+	return;
+}
+
+// Ignore differences in port....
+// must be careful which one we accept after this.
+// can could end-up with the wrong port.
+// However this only matters with firewalled peers anyway.
+// So not too serious.
+bool bdStdSimilarId(const bdId *n1, const bdId *n2)
+{
+	if (n1->id == n2->id)
+	{
+		if (n1->addr.sin_addr.s_addr == n2->addr.sin_addr.s_addr)
+		{
+			return true;
+		}
+	}
+	return false;
+}
+
+bool bdStdUpdateSimilarId(bdId *dest, const bdId *src)
+{
+	/* only difference that's currently allowed */
+	if (dest->addr.sin_port == src->addr.sin_port)
+	{
+		/* no update required */
+		return false;
+	}
+
+	dest->addr.sin_port = src->addr.sin_port;
+	return true;
+}
+
+
+/* fills in bdNodeId r, with XOR of a and b */
+int bdStdDistance(const bdNodeId *a, const bdNodeId *b, bdMetric *r)
+{
+	uint8_t *a_data = (uint8_t *) a->data;
+	uint8_t *b_data = (uint8_t *) b->data;
+	uint8_t *ans = (uint8_t *) r->data;
+	for(int i = 0; i < BITDHT_KEY_LEN; i++)	
+	{
+		*(ans++) = *(a_data++) ^ *(b_data++);
+	}
+	return 1;
+}
+
+void bdStdRandomMidId(const bdNodeId *target, const bdNodeId *other, bdNodeId *midId)
+{
+	bdMetric dist;
+	
+	/* get distance between a & c */
+	bdStdDistance(target, other, &dist);
+
+	/* generate Random Id */
+	bdStdRandomNodeId(midId);
+
+	/* zero bits of Random Id until under 1/2 of distance 
+	 * done in bytes for ease... matches one extra byte than distance = 0 
+	 * -> hence wierd order of operations
+	 */
+	//bool done = false;
+	for(int i = 0; i < BITDHT_KEY_LEN; i++)
+	{
+		midId->data[i] = target->data[i];
+
+		if (dist.data[i] != 0)
+			break;
+	}
+}
+
+int  bdStdLoadNodeId(bdNodeId *id, std::string input)
+{
+	uint8_t *a_data = (uint8_t *) id->data;
+	uint32_t reqlen = BITDHT_KEY_LEN * 2;
+	if (input.size() < reqlen)
+	{
+		return 0;
+	}
+
+	for(int i = 0; i < BITDHT_KEY_LEN; i++)
+	{
+		char ch1 = input[2 * i];
+		char ch2 = input[2 * i + 1];
+		uint8_t value1 = 0;
+		uint8_t value2 = 0;
+
+		/* do char1 */
+        	if (ch1 >= '0' && ch1 <= '9')
+            		value1 = (ch1 - '0');
+        	else if (ch1 >= 'A' && ch1 <= 'F')
+            		value1 = (ch1 - 'A' + 10);
+        	else if (ch1 >= 'a' && ch1 <= 'f')
+            		value1 = (ch1 - 'a' + 10);
+
+		/* do char2 */
+        	if (ch2 >= '0' && ch2 <= '9')
+            		value2 = (ch2 - '0');
+        	else if (ch2 >= 'A' && ch2 <= 'F')
+            		value2 = (ch2 - 'A' + 10);
+        	else if (ch2 >= 'a' && ch2 <= 'f')
+            		value2 = (ch2 - 'a' + 10);
+
+		a_data[i] = (value1 << 4) + value2;
+	}
+	return 1;
+}
+
+std::string bdStdConvertToPrintable(std::string input)
+{
+    std::string out;
+    for(uint32_t i = 0; i < input.length(); i++)
+    {
+        /* sensible chars */
+        if ((input[i] > 31) && (input[i] < 127))
+        {
+                out += input[i];
+        }
+        else
+        {
+            bd_sprintf_append(out, "[0x%x]", (uint32_t) input[i]);
+        }
+    }
+    return out;
+}
+
+void bdStdPrintNodeId(std::ostream &out, const bdNodeId *a)
+{
+	std::string s;
+	bdStdPrintNodeId(s, a, true);
+	out << s;
+}
+
+void bdStdPrintNodeId(std::string &out, const bdNodeId *a, bool append)
+{
+	if (!append)
+	{
+		out.clear();
+	}
+
+	for(int i = 0; i < BITDHT_KEY_LEN; i++)
+	{
+		bd_sprintf_append(out, "%02x", (uint32_t) (a->data)[i]);
+	}
+}
+
+void bdStdPrintId(std::ostream &out, const bdId *a)
+{
+	std::string s;
+	bdStdPrintId(s, a, false);
+	out << s;
+}
+
+void bdStdPrintId(std::string &out, const bdId *a, bool append)
+{
+	bdStdPrintNodeId(out, &(a->id), append);
+	bd_sprintf_append(out, " ip:%s:%u", bdnet_inet_ntoa(a->addr.sin_addr).c_str(), ntohs(a->addr.sin_port));
+}
+
+/* returns 0-160 depending on bucket */
+int bdStdBucketDistance(const bdNodeId *a, const bdNodeId *b)
+{
+	bdMetric m;
+	bdStdDistance(a, b, &m);
+	return bdStdBucketDistance(&m);
+}
+
+/* returns 0-160 depending on bucket */
+int bdStdBucketDistance(const bdMetric *m)
+{
+	for(int i = 0; i < BITDHT_KEY_BITLEN; i++)
+	{
+		int bit = BITDHT_KEY_BITLEN - i - 1;
+		int byte = i / 8;
+		int bbit = 7 - (i % 8);
+		unsigned char comp = (1 << bbit);
+
+#ifdef BITDHT_DEBUG
+		fprintf(stderr, "bdStdBucketDistance: bit:%d  byte:%d bbit:%d comp:%x, data:%x\n", bit, byte, bbit, comp, m->data[byte]);
+#endif
+
+		if (comp & m->data[byte])
+		{
+			return bit;
+		}
+	}
+	return 0;
+}
+
+
+bdStdDht::bdStdDht()
+{
+	return;
+}
+        /* setup variables */
+uint16_t bdStdDht::bdNumBuckets()
+{
+
+	return BITDHT_STANDARD_N_BUCKETS;
+}
+
+uint16_t bdStdDht::bdNodesPerBucket() /* used for bdspace */
+{
+	return BITDHT_STANDARD_BUCKET_SIZE;
+}
+
+uint16_t bdStdDht::bdNumQueryNodes() /* used for queries */
+{
+	return BITDHT_STANDARD_BUCKET_SIZE;
+}
+
+uint16_t bdStdDht::bdBucketBitSize()
+{
+	return BITDHT_STANDARD_BUCKET_SIZE_BITS;
+}
+
+int bdStdDht::bdDistance(const bdNodeId *n1, const bdNodeId *n2, class bdMetric *metric)
+{
+	return bdStdDistance(n1, n2, metric);
+}
+	
+int bdStdDht::bdBucketDistance(const bdNodeId *n1, const bdNodeId *n2)
+{
+	return bdStdBucketDistance(n1, n2);
+}
+	
+int bdStdDht::bdBucketDistance(const bdMetric *metric)
+{
+	return bdStdBucketDistance(metric);
+}
+	
+
+bool bdStdDht::bdSimilarId(const bdId *id1, const bdId *id2)
+{
+	return bdStdSimilarId(id1, id2);
+}
+	
+
+bool bdStdDht::bdUpdateSimilarId(bdId *dest, const bdId *src)
+{
+	return bdStdUpdateSimilarId(dest, src);
+}
+	
+
+void bdStdDht::bdRandomMidId(const bdNodeId *target, const bdNodeId *other, bdNodeId *mid)
+{
+	return bdStdRandomMidId(target, other, mid);
+}
+	
+
+void bdStdDht::bdPrintId(std::ostream &out, const bdId *a)
+{
+	return bdStdPrintId(out, a);
+}
+	
+void bdStdDht::bdPrintNodeId(std::ostream &out, const bdNodeId *a)
+{
+	return bdStdPrintNodeId(out, a);
+}
+
+	
+/**************************/
+
+bdModDht::bdModDht()
+	:mNodesPerBucket(BITDHT_STANDARD_BUCKET_SIZE)
+{
+	return;
+}
+
+void bdModDht::setNodesPerBucket(uint16_t nodesPerBucket)
+{
+	mNodesPerBucket = nodesPerBucket;
+	return;
+}
+
+
+uint16_t bdModDht::bdNodesPerBucket() /* used for bdspace */
+{
+	return mNodesPerBucket;
+}
+
+

libbitdht/src/bitdht/bdstddht.h

@@ -0,0 +1,100 @@
+/*******************************************************************************
+ * bitdht/bdstddht.h                                                           *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2011 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#ifndef BITDHT_STANDARD_DHT_H
+#define BITDHT_STANDARD_DHT_H
+
+#include "bitdht/bdiface.h"
+
+#define BITDHT_STANDARD_BUCKET_SIZE 10 // 20 too many per query?
+#define BITDHT_STANDARD_BUCKET_SIZE_BITS 5
+
+#define BITDHT_STANDARD_N_BUCKETS  BITDHT_KEY_BITLEN
+
+#include <list>
+#include <string>
+#include <map>
+#include <vector>
+
+
+
+
+void bdStdRandomNodeId(bdNodeId *id);
+void bdStdZeroNodeId(bdNodeId *id);
+
+void bdStdRandomId(bdId *id);
+int bdStdDistance(const bdNodeId *a, const bdNodeId *b, bdMetric *r);
+int bdStdBucketDistance(const bdMetric *m);
+int bdStdBucketDistance(const bdNodeId *a, const bdNodeId *b);
+
+void bdStdRandomMidId(const bdNodeId *target, const bdNodeId *other, bdNodeId *mid);
+
+int  bdStdLoadNodeId(bdNodeId *id, std::string input);
+
+void bdStdPrintId(std::ostream &out, const bdId *a);
+void bdStdPrintId(std::string &out, const bdId *a, bool append);
+void bdStdPrintNodeId(std::ostream &out, const bdNodeId *a);
+void bdStdPrintNodeId(std::string &out, const bdNodeId *a, bool append);
+
+std::string bdStdConvertToPrintable(std::string input);
+
+//uint32_t bdStdSimilarNode(const bdId*, const bdId*);
+
+
+class bdStdDht: public bdDhtFunctions
+{
+	public:
+
+        bdStdDht();
+        /* setup variables */
+virtual uint16_t bdNumBuckets();
+virtual uint16_t bdNodesPerBucket(); /* used for bdspace */
+virtual uint16_t bdNumQueryNodes(); /* used for queries */
+virtual uint16_t bdBucketBitSize();
+
+virtual int bdDistance(const bdNodeId *n1, const bdNodeId *n2, bdMetric *metric);
+virtual int bdBucketDistance(const bdNodeId *n1, const bdNodeId *n2);
+virtual int bdBucketDistance(const bdMetric *metric);
+
+virtual bool bdSimilarId(const bdId *id1, const bdId *id2);
+virtual bool bdUpdateSimilarId(bdId *dest, const bdId *src); /* returns true if update was necessary */
+
+virtual void bdRandomMidId(const bdNodeId *target, const bdNodeId *other, bdNodeId *mid);
+
+virtual void bdPrintId(std::ostream &out, const bdId *a);
+virtual void bdPrintNodeId(std::ostream &out, const bdNodeId *a);
+
+};
+
+class bdModDht: public bdStdDht
+{
+	public:
+	bdModDht();
+virtual void setNodesPerBucket(uint16_t nodesPerBucket);
+virtual uint16_t bdNodesPerBucket(); /* used for bdspace */
+
+	private:
+	uint16_t mNodesPerBucket;
+};
+
+
+#endif
+

libbitdht/src/bitdht/bdstore.cc

@@ -0,0 +1,256 @@
+/*******************************************************************************
+ * bitdht/bdstore.cc                                                           *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2011 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "bitdht/bdstore.h"
+#include "util/bdnet.h"
+#include "util/bdfile.h"
+
+#include <stdio.h>
+#include <iostream>
+
+//#define DEBUG_STORE 1
+
+bdStore::bdStore(std::string file, bdDhtFunctions *fns)
+	:mFns(fns)
+{
+#ifdef DEBUG_STORE
+	std::cerr << "bdStore::bdStore(" << file << ")";
+	std::cerr << std::endl;
+#endif
+
+	/* read data from file */
+	mStoreFile = file;
+
+	reloadFromStore();
+}
+
+int bdStore::clear()
+{
+	mIndex = 0;
+	store.clear();
+	return 1;
+}
+
+int bdStore::reloadFromStore()
+{
+	clear();
+
+	FILE *fd = fopen(mStoreFile.c_str(), "r");
+	if (!fd)
+	{
+		fprintf(stderr, "Failed to Open File: %s ... No Peers\n", mStoreFile.c_str());
+		return 0;
+	}
+		
+
+	char line[10240];
+	char addr_str[10240];
+	struct sockaddr_in addr;
+	addr.sin_family = PF_INET;
+	unsigned short port;
+
+	while(line == fgets(line, 10240, fd))
+	{
+		if (2 == sscanf(line, "%s %hd", addr_str, &port))
+		{
+			if (bdnet_inet_aton(addr_str, &(addr.sin_addr)))
+			{
+				addr.sin_port = htons(port);
+				bdPeer peer;
+				bdZeroNodeId(&(peer.mPeerId.id));
+				peer.mPeerId.addr = addr;
+				peer.mLastSendTime = 0;
+				peer.mLastRecvTime = 0;
+				store.push_back(peer);
+#ifdef DEBUG_STORE
+				fprintf(stderr, "Read: %s %d\n", inet_ntoa(addr.sin_addr), ntohs(addr.sin_port));
+#endif
+			}
+		}
+	}
+
+	fclose(fd);
+
+#ifdef DEBUG_STORE
+	fprintf(stderr, "Read %ld Peers\n", (long) store.size());
+#endif
+
+	return 1;
+
+}
+
+// This is a very ugly function!
+int 	bdStore::getPeer(bdPeer *peer)
+{
+#ifdef DEBUG_STORE
+	fprintf(stderr, "bdStore::getPeer() %ld Peers left\n", (long) store.size());
+#endif
+
+	std::list<bdPeer>::iterator it;
+	int i = 0;
+	for(it = store.begin(); (it != store.end()) && (i < mIndex); it++, i++) ; /* empty loop */
+	if (it != store.end())
+	{
+		*peer = *it;
+		mIndex++;
+		return 1;
+	}
+	return 0;
+}
+
+int     bdStore::filterIpList(const std::list<struct sockaddr_in> &filteredIPs)
+{
+	// Nasty O(n^2) iteration over 500 entries!!!. 
+	// hope its not used to often.
+
+	std::list<struct sockaddr_in>::const_iterator it;
+	for(it = filteredIPs.begin(); it != filteredIPs.end(); it++)
+	{
+		std::list<bdPeer>::iterator sit;
+		for(sit = store.begin(); sit != store.end();)
+		{
+			if (it->sin_addr.s_addr == sit->mPeerId.addr.sin_addr.s_addr)
+			{
+				std::cerr << "bdStore::filterIpList() Found Bad entry in Store. Erasing!";
+				std::cerr << std::endl;
+
+				sit = store.erase(sit);
+			}
+			else
+			{
+				sit++;
+			}
+		}
+	}
+	return 1;
+}
+			
+		
+
+#define MAX_ENTRIES 500
+
+	/* maintain a sorted list */
+void	bdStore::addStore(bdPeer *peer)
+{
+#ifdef DEBUG_STORE
+	std::cerr << "bdStore::addStore() ";
+	mFns->bdPrintId(std::cerr, &(peer->mPeerId));
+	std::cerr << std::endl;
+#endif
+
+	/* remove old entry */
+
+	std::list<bdPeer>::iterator it;
+	for(it = store.begin(); it != store.end(); )
+	{
+		if ((it->mPeerId.addr.sin_addr.s_addr == peer->mPeerId.addr.sin_addr.s_addr) &&
+		    (it->mPeerId.addr.sin_port == peer->mPeerId.addr.sin_port))
+		{
+#ifdef DEBUG_STORE
+			std::cerr << "bdStore::addStore() Removed Existing Entry: ";
+			mFns->bdPrintId(std::cerr, &(it->mPeerId));
+			std::cerr << std::endl;
+#endif
+			it = store.erase(it);
+		}
+		else
+		{
+			it++;
+		}
+	}
+
+#ifdef DEBUG_STORE
+	std::cerr << "bdStore::addStore() Push_back";
+	std::cerr << std::endl;
+#endif
+	store.push_back(*peer);
+
+	while(store.size() > MAX_ENTRIES)
+	{
+#ifdef DEBUG_STORE
+		std::cerr << "bdStore::addStore() pop_front()";
+		std::cerr << std::endl;
+#endif
+		store.pop_front();
+	}
+}
+
+void	bdStore::writeStore(std::string file)
+{
+	/* write out store */
+#ifdef DEBUG_STORE
+	fprintf(stderr, "bdStore::writeStore(%s) =  %d entries\n", file.c_str(), store.size());
+#endif
+
+	if (store.size() < 0.9 * MAX_ENTRIES)
+	{
+		/* don't save yet! */
+#ifdef DEBUG_STORE
+		fprintf(stderr, "bdStore::writeStore() Delaying until more entries\n");
+#endif
+		return;
+	}
+
+	std::string filetmp = file + ".tmp" ;
+
+	FILE *fd = fopen(filetmp.c_str(), "w");
+
+	if (!fd)
+	{
+#ifdef DEBUG_STORE
+#endif
+		fprintf(stderr, "bdStore::writeStore() FAILED to Open File\n");
+		return;
+	}
+	
+	std::list<bdPeer>::iterator it;
+	for(it = store.begin(); it != store.end(); it++)
+	{
+		fprintf(fd, "%s %d\n", bdnet_inet_ntoa(it->mPeerId.addr.sin_addr).c_str(), ntohs(it->mPeerId.addr.sin_port));
+#ifdef DEBUG_STORE
+		fprintf(stderr, "Storing Peer Address: %s %d\n", inet_ntoa(it->mPeerId.addr.sin_addr), ntohs(it->mPeerId.addr.sin_port));
+#endif
+
+	}
+	fclose(fd);
+
+	if(!bdFile::renameFile(filetmp,file))
+		std::cerr << "Could not rename file !!" << std::endl;
+#ifdef DEBUG_STORE
+	else
+		std::cerr << "Successfully renamed file " << filetmp << " to " << file << std::endl;
+#endif
+}
+
+void	bdStore::writeStore()
+{
+#if 0
+	if (mStoreFile == "")
+	{
+		return;
+	}
+#endif
+	return writeStore(mStoreFile);
+}
+
+
+

libbitdht/src/bitdht/bdstore.h

@@ -0,0 +1,52 @@
+/*******************************************************************************
+ * bitdht/bdstore.h                                                            *
+ *                                                                             *
+ * BitDHT: An Flexible DHT library.                                            *
+ *                                                                             *
+ * Copyright 2011 by Robert Fernie <bitdht@lunamutt.com>                       *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Affero General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Affero General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Affero General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#ifndef BITDHT_STORE_H
+#define BITDHT_STORE_H
+
+#include <string>
+#include "bitdht/bdiface.h"
+#include "bitdht/bdpeer.h"
+
+class bdStore
+{
+public:
+
+    bdStore(std::string file, bdDhtFunctions *fns);
+
+    int 	reloadFromStore(); /* for restarts */
+    int	filterIpList(const std::list<struct sockaddr_in> &filteredIPs);
+    int 	clear();
+
+    int 	getPeer(bdPeer *peer);
+    void	addStore(bdPeer *peer);
+    void	writeStore(std::string file);
+    void	writeStore();
+
+protected:
+    std::string mStoreFile;
+    std::list<bdPeer> store;
+    int mIndex;
+    bdDhtFunctions *mFns;
+};
+
+
+#endif