Merge pull request #1834 from G10h4ck/android_debug_documentation

Improve RetroShare Android tools and documentation

build_scripts/Android/README.asciidoc

@@ -1,8 +1,8 @@
+= RetroShare development on Android
+
 // SPDX-FileCopyrightText: RetroShare Team <contact@retroshare.cc>
 // SPDX-License-Identifier: CC-BY-SA-4.0
 
-Compile Retroshare for Android
-==============================
 
 Compiling an application for Android is not as easy as one would imagine,
 expecially one like RetroShare that has a big codebase and is not well
@@ -179,6 +179,129 @@ Opening RetroShare android app now should show your old profile.
 
 == Debugging with GDB
 
+If building RetroShare Android package seems tricky, setting up a functional
+debugging environement for it feels like black magic. This section is meant to
+help you doing it with less headache and hopefully in a reproducible way.
+
+Unfortunately at the time of the last update to this guide, Qt build system
+strips debugging symbols from the package and from the buildroot also if you
+compile with debugging enabled. Fiddling with `qmake` configurations and
+variables like `CONFIG+=debug`, `CONFIG+=force_debug_info` or `QMAKE_STRIP`
+either as commandline arguments or inside retroshare `.pro` and `.pri` files is
+uneffective. IMHO Qt should handle this on itself so it is probably worth
+reporting a bug to them. So to workaround this problem you need to fiddle a bit
+with the Android NDK. In my case I always keep +Debug+ or +Release+ suffix in
+my build directory name depending on what kind of build it is, so I use this
+information and modify `llvm-strip` in a way that it will strip only if stripped
+file path doesn't contain +Debug+.
+
+.Modify llvm-strip inside Android NDK
+--------------------------------------------------------------------------------
+## Set ANDROID_NDK_PATH to your Android NDK installation path
+export ANDROID_NDK_PATH="/opt/android-ndk/"
+
+## Define a convenience variable with llvm-strip path
+export ANDROID_NDK_LLVM_STRIP="${ANDROID_NDK_PATH}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip"
+
+## If not existing yer create a backup of the original llvm-strip
+[ -f "${ANDROID_NDK_LLVM_STRIP}.back" ] ||
+	cp "${ANDROID_NDK_LLVM_STRIP}" "${ANDROID_NDK_LLVM_STRIP}.back"
+
+## Create a new llvm-strip that does nothing if the argument path contains Debug
+cat > "${ANDROID_NDK_LLVM_STRIP}" << LLVMSTRIPTRICK
+#!/bin/bash
+
+echo "\${2}" | grep -q Debug ||
+	"${ANDROID_NDK_LLVM_STRIP}.back" --strip-all "\${2}"
+
+LLVMSTRIPTRICK
+
+## Eventually you can revert back simply by running
+# `mv "${ANDROID_NDK_LLVM_STRIP}.back" "${ANDROID_NDK_LLVM_STRIP}"`
+--------------------------------------------------------------------------------
+
+
+To attach to the `retroshare-service` running on Android you need also to pull a
+debugging sysroot out of your device first, RetroShare sources provides an
+helper script to do that.
+
+.Prepare debugging sysroot with helper script
+[source,bash]
+--------------------------------------------------------------------------------
+## Set RetroShare source path
+export RS_SOURCE_DIR="${HOME}/Development/rs-develop"
+
+## Optionally set your device ID first available will be used, you can run
+## `adb devices` to list available devices.
+#export ANDROID_SERIAL="YT013PSPGK"
+
+## Optionally set a path where to save the debugging sysroot otherwise default
+## is used.
+#export DEBUG_SYSROOT="${HOME}/Builds/debug_sysroot/${ANDROID_SERIAL}/"
+
+## Run the helper script
+${RS_SOURCE_DIR}/build_scripts/Android/pull_sysroot.sh
+--------------------------------------------------------------------------------
+
+.Prepare Android NDK GDB configurations
+[source,bash]
+--------------------------------------------------------------------------------
+## Optionally set Qt version variable consistently with your installation
+export QT_VERSION="5.12.4"
+
+## Optionally set Qt architecture variable consistently with Android device
+export QT_ARCH="arm64_v8a"
+
+## Optionally set Qt path variable consistently with your installation
+export QT_DIR="/opt/Qt-${QT_VERSION}/${QT_VERSION}/"
+
+## Optionally set RetroShare buildroot path
+export RS_BUILD_DIR="${HOME}/Builds/RetroShare-Android_for_${QT_ARCH}_Clang_Qt_${QT_VERSION//./_}_android_${QT_ARCH}-Debug/"
+
+## Optionally set gdb config file path
+export GDB_CONFIGS_FILE="${HOME}/Builds/gdb_configs_${QT_ARCH}"
+
+## Generate Android NDK GDB configuration
+${RS_SOURCE_DIR}/build_scripts/Android/generate_gdb_init_commands.sh
+--------------------------------------------------------------------------------
+
+
+You will need to run the following steps everytime you want to debug
+`retroshare-service` on Android.
+
+Make sure `retroshare-service` is running on your connected Android device.
+
+.Run GDB server on your Android device from your host console
+[source,bash]
+--------------------------------------------------------------------------------
+${RS_SOURCE_DIR}/build_scripts/Android/start_gdbserver.sh
+--------------------------------------------------------------------------------
+
+
+.Run Android NDK GDB on your workstation and attach
+[source,bash]
+--------------------------------------------------------------------------------
+## Start NDK gdb
+${ANDROID_NDK_PATH}/prebuilt/linux-x86_64/bin/gdb
+
+## Instruct GDB how and where to find debugging symbols
+(gdb) source $GDB_CONFIGS_FILE
+
+## Connect to the gdbserver running on the phone
+(gdb) target remote 127.0.01:5039
+
+## Have fun debugging
+(gdb)
+--------------------------------------------------------------------------------
+
+
+== Debugging with Qt Creator
+
+WARNING: As of the last update to this guide, debugging retroshare-service
+running on Android via Qt creator doesn't wrok even with all the trickery
+explained in this section, you better learn how to debug with GDB reading
+carefully previous section.
+
 Qt Creator actually support debugging only for the foreground activity, so to
 debug what's happening in the core extra trickery is needed.
 
@@ -239,6 +362,17 @@ TIP: Some time WiFi power saving on Android mess with the GDB connection,
 to prevent that from appening open another +adb shell+ and live +ping+ toward
 your work-station running
 
+== Embedding into other Android packages
+
+As showed by https://elrepo.io/[elRepo.io] developers it is possible and quite
+easy to embed `retroshare-service` into other Android packages see description
+
+https://gitlab.com/elRepo.io/elRepo.io-android/-/blob/master/README.adoc
+
+And implementation details
+
+https://gitlab.com/elRepo.io/elRepo.io-android/-/blob/master/android/app/build.gradle
+
 
 == Furter Readings
 
@@ -254,3 +388,12 @@ your work-station running
 - link:https://fw4spl-org.github.io/fw4spl-blog/2015/07/27/Native-debugging-on-Android-with-QtCreator.html[]
 - link:https://fragglet.livejournal.com/19646.html[]
 - link:https://github.com/android-ndk/ndk/issues/773[How to build without using standalone toolchain?]
+
+== License
+
+Copyright (C) 2016-2020  Gioacchino Mazzurco <gio@eigenlab.org> +
+Copyright (C) 2020       Asociación Civil Altermundi <info@altermundi.net> +
+
+This work is licensed under a Creative Commons Attribution-Share Alike 4.0 International License.
+
+image::https://i.creativecommons.org/l/by-sa/4.0/88x31.png[Creative Commons License, link=http://creativecommons.org/licenses/by-sa/4.0/]

build_scripts/Android/generate_gdb_init_commands.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+# Script to prepare Android NDK GDB configurations to debug retroshare-service
+#
+# Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>
+# Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation, version 3.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>
+
+
+## Define default value for variable, take two arguments, $1 variable name,
+## $2 default variable value, if the variable is not already define define it
+## with default value.
+function define_default_value()
+{
+	VAR_NAME="${1}"
+	DEFAULT_VALUE="${2}"
+
+	[ -z "${!VAR_NAME}" ] && export ${VAR_NAME}="${DEFAULT_VALUE}"
+}
+
+define_default_value QT_VERSION "5.12.4"
+define_default_value QT_ARCH "arm64_v8a"
+define_default_value QT_DIR "/opt/Qt-${QT_VERSION}/${QT_VERSION}/"
+define_default_value ANDROID_SERIAL "$(adb devices | head -n 2 | tail -n 1 | awk '{print $1}')"
+define_default_value RS_BUILD_DIR "${HOME}/Builds/RetroShare-Android_for_${QT_ARCH}_Clang_Qt_${QT_VERSION//./_}_android_${QT_ARCH}-Debug/"
+define_default_value RS_SOURCE_DIR "${HOME}/Development/rs-develop/"
+define_default_value DEBUG_SYSROOT "${HOME}/Builds/debug_sysroot/${ANDROID_SERIAL}/"
+define_default_value GDB_CONFIGS_FILE "${HOME}/Builds/gdb_configs_${QT_ARCH}"
+
+scanDir()
+{
+	find "$1" -type d -not -path '*/\.git/*' | tr '\n' ':' >> $GDB_CONFIGS_FILE
+}
+
+putSeparator()
+{
+	echo >> $GDB_CONFIGS_FILE
+	echo >> $GDB_CONFIGS_FILE
+}
+
+echo "set sysroot ${DEBUG_SYSROOT}" > $GDB_CONFIGS_FILE
+putSeparator
+
+echo "set auto-solib-add on" >> $GDB_CONFIGS_FILE
+echo -n "set solib-search-path " >> $GDB_CONFIGS_FILE
+scanDir "${RS_BUILD_DIR}"
+scanDir "${DEBUG_SYSROOT}"
+scanDir "${QT_DIR}/android_${QT_ARCH}/lib/"
+scanDir "${QT_DIR}/android_${QT_ARCH}/plugins/"
+scanDir "${QT_DIR}/android_${QT_ARCH}/qml/"
+putSeparator
+
+echo -n "directory " >> $GDB_CONFIGS_FILE
+scanDir ${RS_SOURCE_DIR}/jsonapi-generator/src
+scanDir ${RS_SOURCE_DIR}/libbitdht/src
+scanDir ${RS_SOURCE_DIR}/openpgpsdk/src
+scanDir ${RS_SOURCE_DIR}/libretroshare/src
+scanDir ${RS_SOURCE_DIR}/retroshare-service/src
+scanDir ${RS_SOURCE_DIR}/supportlibs/rapidjson/include/
+scanDir ${RS_SOURCE_DIR}/supportlibs/restbed/source/
+scanDir ${RS_SOURCE_DIR}/supportlibs/udp-discovery-cpp/
+scanDir ${RS_SOURCE_DIR}/supportlibs/restbed/dependency/asio/asio/include/
+scanDir ${RS_SOURCE_DIR}/supportlibs/restbed/dependency/catch/include/
+putSeparator
+
+## see https://stackoverflow.com/questions/28972367/gdb-backtrace-without-stopping
+echo "catch signal SIGSEGV" >> $GDB_CONFIGS_FILE
+echo "commands
+   bt
+   continue
+   end" >> $GDB_CONFIGS_FILE
+putSeparator
+
+echo GDB_CONFIGS_FILE=$GDB_CONFIGS_FILE

build_scripts/Android/prepare-toolchain-clang.sh

@@ -2,7 +2,7 @@
 
 # Script to prepare RetroShare Android package building toolchain
 #
-# Copyright (C) 2016-2019  Gioacchino Mazzurco <gio@eigenlab.org>
+# Copyright (C) 2016-2020  Gioacchino Mazzurco <gio@eigenlab.org>
 #
 # This program is free software: you can redistribute it and/or modify it under
 # the terms of the GNU Affero General Public License as published by the

build_scripts/Android/pull_sysroot.sh

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+# Script to pull debugging sysroot from Android devices
+#
+# Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>
+# Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation, version 3.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>
+
+<<"ASCIIDOC"
+
+Pull files from Android device to prepare a debugging sysroot
+Inspired by:
+https://fw4spl-org.github.io/fw4spl-blog/2015/07/27/Native-debugging-on-Android-with-QtCreator.html
+
+The goal is to have a local copy of the sysroot of the connected device.
+For that we use the command `adb pull <remote_dir_or_file> <local_dir_or_file>`
+
+We will get a copy of:
+
+- `/system/lib/`
+- `/vendor/lib/`
+- `libc.so`
+- `libcutils.so`
+
+As well as the binaries `linker` and `app_process`
+
+IMPORTANT:
+from one device to another, the remote file `app_process` can be a binary file
+or a symlink to a binary file - which is NOT ok for us.
+That's so we will try to pull every known possible variants of `app_process`,
+e.g. `app_process32` or `app_process_init`
+
+ASCIIDOC
+
+
+## Define default value for variable, take two arguments, $1 variable name,
+## $2 default variable value, if the variable is not already define define it
+## with default value.
+function define_default_value()
+{
+	VAR_NAME="${1}"
+	DEFAULT_VALUE="${2}"
+
+	[ -z "${!VAR_NAME}" ] && export ${VAR_NAME}="${DEFAULT_VALUE}"
+}
+
+define_default_value ANDROID_SERIAL "$(adb devices | head -n 2 | tail -n 1 | awk '{print $1}')"
+define_default_value DEBUG_SYSROOT "${HOME}/Builds/debug_sysroot/${ANDROID_SERIAL}/"
+
+rm -rf "${DEBUG_SYSROOT}"
+
+for mDir in  "/system/lib/" "/vendor/lib/"; do
+	mkdir -p "${DEBUG_SYSROOT}/${mDir}"
+	# adb pull doesn't behave like rsync dirA/ dirB/ so avoid nesting the
+	# directory by deleting last one before copying
+	rmdir "${DEBUG_SYSROOT}/${mDir}"
+	adb pull "${mDir}" "${DEBUG_SYSROOT}/${mDir}"
+done
+
+# Retrieve the specific binaries - some of these adb commands will fail, since
+# some files may not exist, but that's ok.
+
+mkdir -p "${DEBUG_SYSROOT}/system/bin/"
+for mBin in "/system/bin/linker" "/system/bin/app_process" \
+	"/system/bin/app_process_init" "/system/bin/app_process32" \
+	"/system/bin/app_process64" ; do
+	adb pull "${mBin}" "${DEBUG_SYSROOT}/${mBin}"
+done
+
+# Verify which variants of the specific binaries could be pulled 
+echo
+echo "Found the following specific binaries:"
+echo
+ls -1 "${DEBUG_SYSROOT}/system/bin/"*
+ls -1 "${DEBUG_SYSROOT}/system/lib/libc.so"*
+ls -1 "${DEBUG_SYSROOT}/system/lib/libcutils.so"*
+echo
+
+echo DEBUG_SYSROOT="${DEBUG_SYSROOT}"

build_scripts/Android/start_gdbserver.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# Script to start gdbserver on Android device and attach to retroshare-service
+#
+# Copyright (C) 2020  Gioacchino Mazzurco <gio@eigenlab.org>
+# Copyright (C) 2020  Asociación Civil Altermundi <info@altermundi.net>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the
+# Free Software Foundation, version 3.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>
+
+<<"ASCIIDOC"
+
+Start gdbserver on Android device and attach it to the retroshare service
+process
+
+Inspired by:
+https://fw4spl-org.github.io/fw4spl-blog/2015/07/27/Native-debugging-on-Android-with-QtCreator.html
+
+ASCIIDOC
+
+
+## Define default value for variable, take two arguments, $1 variable name,
+## $2 default variable value, if the variable is not already define define it
+## with default value.
+function define_default_value()
+{
+	VAR_NAME="${1}"
+	DEFAULT_VALUE="${2}"
+
+	[ -z "${!VAR_NAME}" ] && export ${VAR_NAME}="${DEFAULT_VALUE}"
+}
+
+define_default_value ANDROID_APK_PACKAGE "org.retroshare.service"
+define_default_value ANDROID_PROCESS_NAME "org.retroshare.service:rs"
+define_default_value ANDROID_SERIAL "$(adb devices | head -n 2 | tail -n 1 | awk '{print $1}')"
+define_default_value GDB_SERVER_PORT 5039
+
+adb_shell()
+{
+	adb shell run-as ${ANDROID_APK_PACKAGE} $@
+}
+
+## If not passed as environement variable try to determine gdbserver path
+## shipped withing Android package
+[ -z "${LIB_GDB_SERVER_PATH}" ] &&
+{
+	for mUsualPath in \
+		"/data/data/${ANDROID_APK_PACKAGE}/lib/libgdbserver.so" \
+		"/data/app/${ANDROID_APK_PACKAGE}"*"/lib/arm64/libgdbserver.so"
+	do
+		adb_shell ls "${mUsualPath}" &&
+			export LIB_GDB_SERVER_PATH="${mUsualPath}" && break
+	done
+}
+
+
+[ -z "${LIB_GDB_SERVER_PATH}" ] &&
+{
+	cat <<EOF
+libgdbserver.so not found in any of the usual path attempting to look for it
+with find, it will take a bunch of time. Take note of the discovered path and
+define LIB_GDB_SERVER_PATH on your commandline at next run to avoid waiting
+again.
+
+
+EOF
+	tFile="$(mktemp)"
+	adb_shell find -type f -name 'libgdbserver.so' / | \
+		grep ${ANDROID_APK_PACKAGE} | tee "${tFile}"
+
+	LIB_GDB_SERVER_PATH="$(head -n 1 "${tFile}")"
+	rm "${tFile}"
+}
+
+[ -z "${LIB_GDB_SERVER_PATH}" ] &&
+{
+	echo "Cannot find libgdbserver.so, are you sure your package ships it?"
+	exit -1
+}
+
+mPid="$(adb_shell ps | grep ${ANDROID_PROCESS_NAME} | awk '{print $2}')"
+[ -z "${mPid}" ] &&
+{
+	echo "Failed ${ANDROID_PROCESS_NAME} PID retrival are you sure it is running?"
+	exit -2
+}
+
+
+## Establish port forwarding so we can connect to gdbserver with gdb
+adb forward tcp:${GDB_SERVER_PORT} tcp:${GDB_SERVER_PORT}
+
+((adb_shell ${LIB_GDB_SERVER_PATH} 127.0.0.1:${GDB_SERVER_PORT} --attach ${mPid})&)