Merge pull request #2466 from G10h4ck/android_CI

Android: modernize toolchain, provide Dockerfile and setup GitlabCI
This commit is contained in:
G10h4ck 2021-09-01 21:56:52 +02:00 committed by GitHub
commit 9b1568a1ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 775 additions and 229 deletions

View File

@ -51,3 +51,59 @@ build-and-test:
docker exec retroshare
curl --verbose http://127.0.0.1:9092/rsMsgs/getChatLobbyList | jq
- docker container stop retroshare
build-android-arm-apk:
script:
- >
if [ -n "$CI_MERGE_REQUEST_ID" ]; then
REPO_ARGS="--build-arg REPO_URL=$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" ;
REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ;
else
REPO_ARGS="--build-arg REPO_URL=$CI_REPOSITORY_URL" ;
REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_COMMIT_BRANCH" ;
fi ;
export REPO_ARGS ;
echo REPO_ARGS=$REPO_ARGS ;
- mkdir Dockercontext
- >
docker build -t retroshare:android_arm_latest $REPO_ARGS
--build-arg ANDROID_PLATFORM_VER=16
--build-arg ANDROID_NDK_ARCH=arm
--file $CI_PROJECT_DIR/build_scripts/GitlabCI/Android.Dockerfile
Dockercontext
# see https://stackoverflow.com/a/59055906
- >
docker cp
$(docker create --rm retroshare:android_arm_latest):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk
$CI_PROJECT_DIR/RetroShare_Android_Service-arm.apk
artifacts:
paths:
- RetroShare_Android_Service-arm.apk
# Use separate runner to avoid no space left on device
build-android-arm64-apk:
script:
- >
if [ -n "$CI_MERGE_REQUEST_ID" ]; then
REPO_ARGS="--build-arg REPO_URL=$CI_MERGE_REQUEST_SOURCE_PROJECT_URL" ;
REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" ;
else
REPO_ARGS="--build-arg REPO_URL=$CI_REPOSITORY_URL" ;
REPO_ARGS="$REPO_ARGS --build-arg REPO_BRANCH=$CI_COMMIT_BRANCH" ;
fi ;
export REPO_ARGS ;
echo REPO_ARGS=$REPO_ARGS ;
- mkdir Dockercontext
- >
docker build -t retroshare:android_arm64_latest $REPO_ARGS
--build-arg ANDROID_PLATFORM_VER=21
--build-arg ANDROID_NDK_ARCH=arm64
--file $CI_PROJECT_DIR/build_scripts/GitlabCI/Android.Dockerfile
Dockercontext
- >
docker cp
$(docker create --rm retroshare:android_arm64_latest):/retroshare-service-android-build/android-build/build/outputs/apk/debug/android-build-debug.apk
$CI_PROJECT_DIR/RetroShare_Android_Service-arm64.apk
artifacts:
paths:
- RetroShare_Android_Service-arm64.apk

View File

@ -9,7 +9,7 @@ RetroShare provides file sharing, chat, messages, forums, channels and more.
.Build Status
|===============================================================================
|GNU/Linux (via Gitlab CI) | image:https://gitlab.com/RetroShare/RetroShare/badges/master/pipeline.svg[link="https://gitlab.com/RetroShare/RetroShare/-/commits/master",title="pipeline status"]
|GNU/Linux, Android (via Gitlab CI) | image:https://gitlab.com/RetroShare/RetroShare/badges/master/pipeline.svg[link="https://gitlab.com/RetroShare/RetroShare/-/commits/master",title="pipeline status"]
|GNU/Linux, macOS, (via Travis CI) | image:https://travis-ci.org/RetroShare/RetroShare.svg?branch=master[link="https://travis-ci.org/RetroShare/RetroShare"]
|Windows (via AppVeyor) | image:https://ci.appveyor.com/api/projects/status/github/RetroShare/RetroShare?svg=true[link="https://ci.appveyor.com/project/RetroShare58622/retroshare"]
|===============================================================================

View File

@ -0,0 +1,118 @@
## To prepare an image suitable as base for Gitlab CI use
## image name must match gitlab repository name, you can play just with the tag
## the part after :
# export CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_arm_base"
# docker build --squash -t "${CI_REGISTRY_IMAGE}" \
# --build-arg QT_INSTALLER_JWT_TOKEN="your qt JWT token goes here" .
#
# To build Android ARMv8 (64 bit) package pass also
# export CI_IMAGE_NAME="registry.gitlab.com/retroshare/retroshare:android_arm64_base"
# --build-arg ANDROID_NDK_ARCH=arm64 --build-arg ANDROID_PLATFORM_VER=21
## --squash is very important in case of GitlabCI shared runners as they are
## limited to 25GB disk size
## To push it to gitlab CI registry you need first to login and the to push
# docker login registry.gitlab.com
# docker push ${CI_IMAGE_NAME}
FROM ubuntu:20.04
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get clean
RUN apt-get install -y -qq \
bash build-essential bzip2 cmake curl chrpath doxygen \
git p7zip python qt5-default qttools5-dev tclsh unzip wget zip
# Dependencies to create Android pkg
RUN apt-get install -y -qq \
openjdk-8-jre openjdk-8-jdk openjdk-8-jdk-headless gradle
ARG FRESHCLONE=0
ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
ARG REPO_BRANCH=master
ARG REPO_DEPTH="--depth 2000"
RUN git clone $REPO_DEPTH $REPO_URL -b $REPO_BRANCH && cd RetroShare && \
git fetch --tags
ENV PREPARE_TOOLCHAIN="/RetroShare/build_scripts/Android/prepare-toolchain-clang.sh"
ENV NATIVE_LIBS_TOOLCHAIN_PATH="/android-toolchain/"
ARG ANDROID_PLATFORM_VER=16
ARG ANDROID_NDK_ARCH=arm
ENV ANDROID_SDK_PATH="/opt/android-sdk"
ENV ANDROID_HOME="$ANDROID_SDK_PATH"
ENV ANDROID_SDK_ROOT="$ANDROID_SDK_PATH"
ENV ANDROID_NDK_PATH="/opt/android-ndk"
ENV ANDROID_NDK_ROOT="$ANDROID_NDK_PATH"
ENV PATH="$PATH:$ANDROID_HOME/tools"
ENV PATH="$PATH:$ANDROID_HOME/platform-tools"
ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
RUN mkdir /bCache
WORKDIR /bCache
## Quick HACK to ease development
#COPY prepare-toolchain-clang.sh $PREPARE_TOOLCHAIN
RUN $PREPARE_TOOLCHAIN install_android_sdk
RUN $PREPARE_TOOLCHAIN install_android_ndk
RUN $PREPARE_TOOLCHAIN bootstrap_toolchain
RUN $PREPARE_TOOLCHAIN build_bzlib
RUN $PREPARE_TOOLCHAIN build_openssl
RUN $PREPARE_TOOLCHAIN build_sqlite
RUN $PREPARE_TOOLCHAIN build_sqlcipher
RUN $PREPARE_TOOLCHAIN build_rapidjson
RUN $PREPARE_TOOLCHAIN build_restbed
RUN $PREPARE_TOOLCHAIN build_udp-discovery-cpp
RUN $PREPARE_TOOLCHAIN build_xapian
RUN $PREPARE_TOOLCHAIN build_miniupnpc
RUN $PREPARE_TOOLCHAIN deduplicate_includes
ARG QT_INSTALLER_JWT_TOKEN
RUN $PREPARE_TOOLCHAIN install_qt_android
# Avoid Qt account details leak into the image
RUN rm -f /root/.local/share/Qt/qtaccount.ini
# Shrink image by removing unneded Qt components
RUN rm -r \
$NATIVE_LIBS_TOOLCHAIN_PATH/Qt/Docs/ \
$NATIVE_LIBS_TOOLCHAIN_PATH/Qt/Examples/ \
$NATIVE_LIBS_TOOLCHAIN_PATH/Qt/Tools/
RUN mkdir /jsonapi-generator-build
WORKDIR /jsonapi-generator-build/
RUN qmake ../RetroShare/jsonapi-generator/src/ \
CONFIG+=no_retroshare_plugins \
CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
CONFIG+=rs_jsonapi && \
make -j$(nproc)
#CONFIG+=no_keywords
RUN mkdir /retroshare-service-android-build
WORKDIR /retroshare-service-android-build
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
-spec android-clang \
CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
RS_UPNP_LIB=miniupnpc \
JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
CONFIG+=no_rs_service_terminal_login
RUN make -j$(nproc)
RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/androiddeployqt \
--input retroshare-service/src/android-libretroshare-service.so-deployment-settings.json \
--output android-build --android-platform android-$ANDROID_PLATFORM_VER \
--jdk $JAVA_HOME --gradle
RUN rm -rf /bCache
# Clean apt cache
RUN apt-get clean && rm -rf /var/lib/apt/lists/*

View File

@ -2,7 +2,8 @@
# Script to prepare RetroShare Android package building toolchain
#
# Copyright (C) 2016-2020 Gioacchino Mazzurco <gio@eigenlab.org>
# Copyright (C) 2016-2021 Gioacchino Mazzurco <gio@eigenlab.org>
# Copyright (C) 2020-2021 Asociación Civil Altermundi <info@altermundi.net>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the
@ -38,6 +39,15 @@ define_default_value ANDROID_PLATFORM_VER "16"
define_default_value NATIVE_LIBS_TOOLCHAIN_PATH "${HOME}/Builds/android-toolchains/retroshare-android-${ANDROID_PLATFORM_VER}-${ANDROID_NDK_ARCH}/"
define_default_value HOST_NUM_CPU $(nproc)
define_default_value ANDROID_SDK_INSTALL "false"
define_default_value ANDROID_SDK_TOOLS_VERSION "3859397"
define_default_value ANDROID_SDK_TOOLS_SHA256 444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0
define_default_value ANDROID_SDK_VERSION "29.0.3"
define_default_value ANDROID_NDK_INSTALL "false"
define_default_value ANDROID_NDK_VERSION "r21"
define_default_value ANDROID_NDK_SHA256 b65ea2d5c5b68fb603626adcbcea6e4d12c68eb8a73e373bbb9d23c252fc647b
define_default_value BZIP2_SOURCE_VERSION "1.0.6"
define_default_value BZIP2_SOURCE_SHA256 a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
@ -48,16 +58,22 @@ define_default_value SQLITE_SOURCE_YEAR "2018"
define_default_value SQLITE_SOURCE_VERSION "3250200"
define_default_value SQLITE_SOURCE_SHA256 da9a1484423d524d3ac793af518cdf870c8255d209e369bd6a193e9f9d0e3181
define_default_value SQLCIPHER_SOURCE_VERSION "4.2.0"
define_default_value SQLCIPHER_SOURCE_SHA256 105c1b813f848da038c03647a8bfc9d42fb46865e6aaf4edfd46ff3b18cdccfc
define_default_value SQLCIPHER_SOURCE_VERSION "4.4.3"
define_default_value SQLCIPHER_SOURCE_SHA256 b8df69b998c042ce7f8a99f07cf11f45dfebe51110ef92de95f1728358853133
define_default_value LIBUPNP_SOURCE_VERSION "1.8.4"
define_default_value LIBUPNP_SOURCE_SHA256 976c3e4555604cdd8391ed2f359c08c9dead3b6bf131c24ce78e64d6669af2ed
define_default_value INSTALL_QT_ANDROID "false"
define_default_value QT_VERSION "5.12.0"
define_default_value QT_ANDROID_VIA_INSTALLER "false"
define_default_value QT_VERSION "5.12.11"
define_default_value QT_INSTALLER_VERSION "4.1.1"
define_default_value QT_INSTALLER_SHA256 1266ffd0d1b0e466244e3bc8422975c1aa9d96745b6bb28d422f7f92df11f34c
define_default_value QT_INSTALLER_JWT_TOKEN "Need a QT account JWT token to use the insaller see https://wiki.qt.io/Online_Installer_4.x"
define_default_value QT_INSTALL_PATH "${NATIVE_LIBS_TOOLCHAIN_PATH}/Qt/"
define_default_value QT_ANDROID_INSTALLER_SHA256 a214084e2295c9a9f8727e8a0131c37255bf724bfc69e80f7012ba3abeb1f763
define_default_value RESTBED_SOURCE_REPO "https://github.com/Corvusoft/restbed.git"
define_default_value RESTBED_SOURCE_VERSION f74f9329dac82e662c1d570b7cd72c192b729eb4
define_default_value UDP_DISCOVERY_CPP_SOURCE "https://github.com/truvorskameikin/udp-discovery-cpp.git"
@ -72,6 +88,64 @@ define_default_value RAPIDJSON_SOURCE_SHA256 bf7ced29704a1e696fbccf2a2b4ea068e77
define_default_value MINIUPNPC_SOURCE_VERSION "2.1.20190625"
define_default_value MINIUPNPC_SOURCE_SHA256 8723f5d7fd7970de23635547700878cd29a5c2bb708b5e5475b2d1d2510317fb
# zlib and libpng versions walks toghether
define_default_value ZLIB_SOURCE_VERSION "1.2.11"
define_default_value ZLIB_SOURCE_SHA256 4ff941449631ace0d4d203e3483be9dbc9da454084111f97ea0a2114e19bf066
define_default_value LIBPNG_SOURCE_VERSION "1.6.37"
define_default_value LIBPNG_SOURCE_SHA256 505e70834d35383537b6491e7ae8641f1a4bed1876dbfe361201fc80868d88ca
define_default_value LIBJPEG_SOURCE_VERSION "9d"
define_default_value LIBJPEG_SOURCE_SHA256 6c434a3be59f8f62425b2e3c077e785c9ce30ee5874ea1c270e843f273ba71ee
define_default_value TIFF_SOURCE_VERSION "4.2.0"
define_default_value TIFF_SOURCE_SHA256 eb0484e568ead8fa23b513e9b0041df7e327f4ee2d22db5a533929dfc19633cb
define_default_value CIMG_SOURCE_VERSION "2.9.7"
define_default_value CIMG_SOURCE_SHA256 595dda9718431a123b418fa0db88e248c44590d47d9b1646970fa0503e27fa5c
define_default_value PHASH_SOURCE_REPO "https://gitlab.com/g10h4ck/pHash.git"
define_default_value PHASH_SOURCE_VERSION origin/android-ndk
define_default_value MVPTREE_SOURCE_REPO "https://github.com/starkdg/mvptree.git"
define_default_value MVPTREE_SOURCE_VERSION origin/master
define_default_value REPORT_DIR "$(pwd)/$(basename ${NATIVE_LIBS_TOOLCHAIN_PATH})_build_report/"
cArch=""
eABI=""
cmakeABI=""
case "${ANDROID_NDK_ARCH}" in
"arm")
cArch="${ANDROID_NDK_ARCH}"
eABI="eabi"
;;
"arm64")
cArch="aarch64"
eABI=""
;;
"x86")
cArch="i686"
eABI=""
;;
"x86_64")
echo "ANDROID_NDK_ARCH=${ANDROID_NDK_ARCH} not supported yet"
exit -1
cArch="??"
eABI=""
esac
export SYSROOT="${NATIVE_LIBS_TOOLCHAIN_PATH}/sysroot/"
export PREFIX="${SYSROOT}/usr/"
export CC="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang"
export CXX="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang++"
export AR="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ar"
export RANLIB="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ranlib"
# Used to instruct cmake to explicitely ignore host libraries
export HOST_IGNORE_PREFIX="/usr/"
## $1 filename, $2 sha256 hash
function check_sha256()
@ -104,159 +178,262 @@ function verified_download()
}
}
cArch=""
eABI=""
# This function is the result of reading and testing many many stuff be very
# careful editing it
function andro_cmake()
{
# Using android.toolchain.cmake as documented here
# https://developer.android.com/ndk/guides/cmake seens to break more things then
# it fixes :-\
cmakeProc=""
case "${ANDROID_NDK_ARCH}" in
"arm")
cArch="${ANDROID_NDK_ARCH}"
eABI="eabi"
cmakeProc="armv7-a"
;;
"arm64")
cArch="aarch64"
eABI=""
cmakeProc="aarch64"
;;
"x86")
cArch="i686"
eABI=""
cmakeProc="i686"
;;
"x86_64")
cmakeProc="x86_64"
;;
*)
echo "Unhandled NDK architecture ${ANDROID_NDK_ARCH}"
exit -1
;;
esac
export SYSROOT="${NATIVE_LIBS_TOOLCHAIN_PATH}/sysroot/"
export PREFIX="${SYSROOT}/usr/"
export CC="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang"
export CXX="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-clang++"
export AR="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ar"
export RANLIB="${NATIVE_LIBS_TOOLCHAIN_PATH}/bin/${cArch}-linux-android${eABI}-ranlib"
_hi="$HOST_IGNORE_PREFIX"
cmake \
-DCMAKE_SYSTEM_PROCESSOR=$cmakeProc \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DCMAKE_PREFIX_PATH="${PREFIX}" \
-DCMAKE_SYSTEM_PREFIX_PATH="${PREFIX}" \
-DCMAKE_INCLUDE_PATH="${PREFIX}/include" \
-DCMAKE_SYSTEM_INCLUDE_PATH="${PREFIX}/include" \
-DCMAKE_LIBRARY_PATH="${PREFIX}/lib" \
-DCMAKE_SYSTEM_LIBRARY_PATH="${PREFIX}/lib" \
-DCMAKE_INSTALL_PREFIX="${PREFIX}" \
-DCMAKE_IGNORE_PATH="$_hi/include;$_hi/lib;$_hi/lib64" \
$@
# It is probably ok to do not touch CMAKE_PROGRAM_PATH and
# CMAKE_SYSTEM_PROGRAM_PATH
}
function git_source_get()
{
sourceDir="$1" ; shift #$1
sourceRepo="$1" ; shift #$2
sourceVersion="$1" ; shift #$3
# extra paramethers are treated as submodules
[ -d "$sourceDir" ] &&
{
pushd "$sourceDir"
actUrl="$(git remote get-url origin)"
[ "$actUrl" != "$sourceRepo" ] && rm -rf "${sourceDir}"
popd
} || true
[ -d $sourceDir ] || git clone "$sourceRepo" "$sourceDir"
pushd $sourceDir
git fetch --all
git reset --hard ${sourceVersion}
while [ "$1" != "" ] ; do
git submodule update --init "$1"
pushd "$1"
git reset --hard
shift
popd
done
popd
}
declare -A TASK_REGISTER
function task_register()
{
TASK_REGISTER[$1]=true
}
function task_unregister()
{
# we may simply wipe them but we could benefit from keeping track of
# unregistered tasks too
TASK_REGISTER[$1]=false
}
function task_logfile()
{
echo "$REPORT_DIR/$1.log"
}
function task_run()
{
mTask="$1" ; shift
[ "${TASK_REGISTER[$mTask]}" != "true" ] &&
{
echo "Attempt to run not registered task $mTask $@"
return -1
}
logFile="$(task_logfile $mTask)"
if [ -f "$logFile" ] ; then
echo "Task $mTask already run more details at $logFile"
else
date | tee > "$logFile"
$mTask $@ |& tee --append "$logFile"
mRetval="${PIPESTATUS[0]}"
echo "Task $mTask return ${mRetval} more details at $logFile"
date | tee --append "$logFile"
return ${mRetval}
fi
}
function task_zap()
{
rm -f "$(task_logfile $1)"
}
DUPLICATED_INCLUDES_LIST_FILE="${REPORT_DIR}/duplicated_includes_list"
DUPLICATED_INCLUDES_DIR="${REPORT_DIR}/duplicated_includes/"
task_register install_android_sdk
install_android_sdk()
{
tFile="sdk-tools-linux-${ANDROID_SDK_TOOLS_VERSION}.zip"
verified_download "${tFile}" "${ANDROID_SDK_TOOLS_SHA256}" \
"https://dl.google.com/android/repository/${tFile}"
unzip "${tFile}"
mkdir -p "$ANDROID_SDK_PATH"
rm -rf "$ANDROID_SDK_PATH/tools/"
mv --verbose tools/ "$ANDROID_SDK_PATH/tools/"
# Install Android SDK
yes | $ANDROID_SDK_PATH/tools/bin/sdkmanager --licenses && \
$ANDROID_SDK_PATH/tools/bin/sdkmanager --update
$ANDROID_SDK_PATH/tools/bin/sdkmanager "platforms;android-$ANDROID_PLATFORM_VER"
$ANDROID_SDK_PATH/tools/bin/sdkmanager "build-tools;$ANDROID_SDK_VERSION"
}
task_register install_android_ndk
install_android_ndk()
{
tFile="android-ndk-${ANDROID_NDK_VERSION}-linux-x86_64.zip"
verified_download "${tFile}" "${ANDROID_NDK_SHA256}" \
"https://dl.google.com/android/repository/${tFile}"
unzip "${tFile}"
mkdir -p "$ANDROID_NDK_PATH"
rm -rf "$ANDROID_NDK_PATH"
mv --verbose "android-ndk-${ANDROID_NDK_VERSION}/" "$ANDROID_NDK_PATH/"
}
## More information available at https://android.googlesource.com/platform/ndk/+/ics-mr0/docs/STANDALONE-TOOLCHAIN.html
build_toolchain()
task_register bootstrap_toolchain
bootstrap_toolchain()
{
echo "build_toolchain()
################################################################################
################################################################################
################################################################################
"
rm -rf ${NATIVE_LIBS_TOOLCHAIN_PATH}
rm -rf "${NATIVE_LIBS_TOOLCHAIN_PATH}"
${ANDROID_NDK_PATH}/build/tools/make_standalone_toolchain.py --verbose \
--arch ${ANDROID_NDK_ARCH} --install-dir ${NATIVE_LIBS_TOOLCHAIN_PATH} \
--api ${ANDROID_PLATFORM_VER}
find "${PREFIX}/include/" -not -type d > "${NATIVE_LIBS_TOOLCHAIN_PATH}/deletefiles"
# Avoid problems with arm64 some libraries installing on lib64
ln -s "${PREFIX}/lib/" "${PREFIX}/lib64"
pushd "${PREFIX}/include/"
find . -not -type d > "${DUPLICATED_INCLUDES_LIST_FILE}"
popd
}
## This avoid <cmath> include errors due to -isystem and -I ordering issue
delete_copied_includes()
task_register deduplicate_includes
deduplicate_includes()
{
echo "delete_copied_includes()
################################################################################
################################################################################
################################################################################
"
cat "${NATIVE_LIBS_TOOLCHAIN_PATH}/deletefiles" | while read delFile ; do
rm "$delFile"
done
while read delFile ; do
mNewPath="${DUPLICATED_INCLUDES_DIR}/$delFile"
mkdir --verbose --parents "$(dirname "$mNewPath")"
mv --verbose "${PREFIX}/include/$delFile" "$mNewPath"
done < "${DUPLICATED_INCLUDES_LIST_FILE}"
}
## More information available at https://gitlab.com/relan/provisioners/merge_requests/1 and http://stackoverflow.com/a/34032216
task_register reduplicate_includes
reduplicate_includes()
{
pushd "${DUPLICATED_INCLUDES_DIR}"
find . -not -type d | while read delFile ; do
mv --verbose "${delFile}" "${PREFIX}/include/$delFile"
done
popd
}
# $1 optional prefix prepended only if return value is not empty
# $2 optional suffix appended only if return value is not empty
task_register get_qt_arch
get_qt_arch()
{
local QT_VERSION_COMP="$(echo $QT_VERSION | awk -F. '{print $1*1000000+$2*1000+$3}')"
local QT_ARCH=""
# Qt >= 5.15.0 ships all Android architectures toghether
[ "$QT_VERSION_COMP" -lt "5015000" ] &&
{
case "${ANDROID_NDK_ARCH}" in
"arm")
QT_ARCH="armv7"
;;
"arm64")
QT_ARCH="arm64_v8a"
;;
"x86")
QT_ARCH="x86"
;;
esac
echo "$1$QT_ARCH$2"
}
}
task_register get_qt_dir
get_qt_dir()
{
echo "${QT_INSTALL_PATH}/${QT_VERSION}/android$(get_qt_arch _)/"
}
## More information available at https://wiki.qt.io/Online_Installer_4.x
task_register install_qt_android
install_qt_android()
{
echo "install_qt_android()
################################################################################
################################################################################
################################################################################
"
QT_VERSION_CODE=$(echo $QT_VERSION | tr -d .)
QT_INSTALL_PATH=${NATIVE_LIBS_TOOLCHAIN_PATH}/Qt
QT_INSTALLER="qt-unified-linux-x64-3.0.2-online.run"
verified_download $QT_INSTALLER $QT_ANDROID_INSTALLER_SHA256 \
http://master.qt.io/archive/online_installers/3.0/${QT_INSTALLER}
QT_VERSION_CODE="$(echo $QT_VERSION | tr -d .)"
QT_INSTALLER="qt-unified-linux-x86_64-${QT_INSTALLER_VERSION}-online.run"
tMajDotMinVer="$(echo $QT_INSTALLER_VERSION | awk -F. '{print $1"."$2}')"
verified_download $QT_INSTALLER $QT_INSTALLER_SHA256 \
"https://master.qt.io/archive/online_installers/${tMajDotMinVer}/${QT_INSTALLER}"
chmod a+x ${QT_INSTALLER}
QT_INSTALLER_SCRIPT="qt_installer_script.js"
cat << EOF > "${QT_INSTALLER_SCRIPT}"
function Controller() {
installer.autoRejectMessageBoxes();
installer.installationFinished.connect(function() {
gui.clickButton(buttons.NextButton);
});
var welcomePage = gui.pageWidgetByObjectName("WelcomePage");
welcomePage.completeChanged.connect(function() {
if (gui.currentPageWidget().objectName == welcomePage.objectName)
gui.clickButton(buttons.NextButton);
});
}
Controller.prototype.WelcomePageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.CredentialsPageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.IntroductionPageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.TargetDirectoryPageCallback = function() {
gui.currentPageWidget().TargetDirectoryLineEdit.setText("$QT_INSTALL_PATH");
gui.clickButton(buttons.NextButton);
}
Controller.prototype.ComponentSelectionPageCallback = function() {
var widget = gui.currentPageWidget();
// You can get these component names by running the installer with the
// --verbose flag. It will then print out a resource tree.
widget.deselectComponent("qt.tools.qtcreator");
widget.deselectComponent("qt.tools.doc");
widget.deselectComponent("qt.tools.examples");
widget.selectComponent("qt.$QT_VERSION_CODE.android_armv7");
gui.clickButton(buttons.NextButton);
}
Controller.prototype.LicenseAgreementPageCallback = function() {
gui.currentPageWidget().AcceptLicenseRadioButton.setChecked(true);
gui.clickButton(buttons.NextButton);
}
Controller.prototype.StartMenuDirectoryPageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.ReadyForInstallationPageCallback = function() {
gui.clickButton(buttons.NextButton);
}
Controller.prototype.FinishedPageCallback = function() {
var checkBoxForm = gui.currentPageWidget().LaunchQtCreatorCheckBoxForm;
if (checkBoxForm && checkBoxForm.launchQtCreatorCheckBox)
checkBoxForm.launchQtCreatorCheckBox.checked = false;
gui.clickButton(buttons.FinishButton);
}
EOF
QT_QPA_PLATFORM=minimal ./${QT_INSTALLER} --script ${QT_INSTALLER_SCRIPT}
QT_QPA_PLATFORM=minimal ./${QT_INSTALLER} \
install qt.qt5.${QT_VERSION_CODE}.android$(get_qt_arch _) \
--accept-licenses --accept-obligations --confirm-command \
--default-answer --no-default-installations \
--root "${QT_INSTALL_PATH}"
}
## More information available at retroshare://file?name=Android%20Native%20Development%20Kit%20Cookbook.pdf&size=29214468&hash=0123361c1b14366ce36118e82b90faf7c7b1b136
task_register build_bzlib
build_bzlib()
{
echo "build_bzlib()
################################################################################
################################################################################
################################################################################
"
B_dir="bzip2-${BZIP2_SOURCE_VERSION}"
rm -rf $B_dir
@ -279,14 +456,9 @@ build_bzlib()
}
## More information available at http://doc.qt.io/qt-5/opensslsupport.html
task_register build_openssl
build_openssl()
{
echo "build_openssl()
################################################################################
################################################################################
################################################################################
"
B_dir="openssl-${OPENSSL_SOURCE_VERSION}"
rm -rf $B_dir
@ -308,6 +480,10 @@ build_openssl()
--openssldir="${SYSROOT}/etc/ssl"
# sed -i 's/LIBNAME=$$i LIBVERSION=$(SHLIB_MAJOR).$(SHLIB_MINOR) \\/LIBNAME=$$i \\/g' Makefile
# sed -i '/LIBCOMPATVERSIONS=";$(SHLIB_VERSION_HISTORY)" \\/d' Makefile
# Avoid documentation build which is unneded and time consuming
echo "exit 0; " > util/process_docs.pl
make -j${HOST_NUM_CPU}
make install
rm -f ${PREFIX}/lib/libssl.so*
@ -315,14 +491,9 @@ build_openssl()
cd ..
}
task_register build_sqlite
build_sqlite()
{
echo "build_sqlite()
################################################################################
################################################################################
################################################################################
"
B_dir="sqlite-autoconf-${SQLITE_SOURCE_VERSION}"
rm -rf $B_dir
@ -335,18 +506,13 @@ build_sqlite()
make -j${HOST_NUM_CPU}
make install
rm -f ${PREFIX}/lib/libsqlite3.so*
# ${CC} -shared -o libsqlite3.so -fPIC sqlite3.o -ldl
# cp libsqlite3.so "${SYSROOT}/usr/lib"
cd ..
}
task_register build_sqlcipher
build_sqlcipher()
{
echo "build_sqlcipher()
################################################################################
################################################################################
################################################################################
"
task_run build_sqlite
B_dir="sqlcipher-${SQLCIPHER_SOURCE_VERSION}"
rm -rf $B_dir
@ -358,14 +524,14 @@ build_sqlcipher()
tar -xf $T_file
cd $B_dir
case "${ANDROID_NDK_ARCH}" in
"arm64")
# SQLCipher config.sub is outdated and doesn't recognize newer architectures
rm config.sub
autoreconf --verbose --install --force
automake --add-missing --copy --force-missing
;;
esac
# case "${ANDROID_NDK_ARCH}" in
# "arm64")
# # SQLCipher config.sub is outdated and doesn't recognize newer architectures
# rm config.sub
# autoreconf --verbose --install --force
# automake --add-missing --copy --force-missing
# ;;
# esac
./configure --with-pic --build=$(sh ./config.guess) \
--host=${cArch}-linux \
--prefix="${PREFIX}" --with-sysroot="${SYSROOT}" \
@ -377,14 +543,9 @@ build_sqlcipher()
cd ..
}
task_register build_libupnp
build_libupnp()
{
echo "build_libupnp()
################################################################################
################################################################################
################################################################################
"
B_dir="pupnp-release-${LIBUPNP_SOURCE_VERSION}"
B_ext=".tar.gz"
B_file="${B_dir}${B_ext}"
@ -408,14 +569,9 @@ build_libupnp()
cd ..
}
task_register build_rapidjson
build_rapidjson()
{
echo "build_rapidjson()
################################################################################
################################################################################
################################################################################
"
B_dir="rapidjson-${RAPIDJSON_SOURCE_VERSION}"
D_file="${B_dir}.tar.gz"
verified_download $D_file $RAPIDJSON_SOURCE_SHA256 \
@ -424,42 +580,25 @@ build_rapidjson()
cp -r "${B_dir}/include/rapidjson/" "${PREFIX}/include/rapidjson"
}
task_register build_restbed
build_restbed()
{
echo "build_restbed()
################################################################################
################################################################################
################################################################################
"
S_dir="restbed"
B_dir="${S_dir}-build"
git_source_get "$S_dir" "$RESTBED_SOURCE_REPO" "${RESTBED_SOURCE_VERSION}" \
"dependency/asio" "dependency/catch"
[ -d restbed ] || git clone --depth=2000 https://github.com/Corvusoft/restbed.git
cd restbed
# git fetch --tags
# git checkout tags/${RESTBED_SOURCE_VERSION}
git checkout ${RESTBED_SOURCE_VERSION}
git submodule update --init dependency/asio
git submodule update --init dependency/catch
git submodule update --init dependency/kashmir
cd ..
rm -rf restbed-build; mkdir restbed-build ; cd restbed-build
cmake \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON BUILD_TESTS=OFF \
-DBUILD_SSL=OFF -DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -H../restbed
rm -rf "$B_dir"; mkdir "$B_dir"
pushd "$B_dir"
andro_cmake -DBUILD_TESTS=OFF -DBUILD_SSL=OFF -B. -H../${S_dir}
make -j${HOST_NUM_CPU}
make install
cp "${PREFIX}/library/librestbed.a" "${PREFIX}/lib/"
cd ..
popd
}
task_register build_udp-discovery-cpp
build_udp-discovery-cpp()
{
echo "build_udp-discovery-cpp()
################################################################################
################################################################################
################################################################################
"
S_dir="udp-discovery-cpp"
[ -d $S_dir ] || git clone $UDP_DISCOVERY_CPP_SOURCE $S_dir
cd $S_dir
@ -468,23 +607,16 @@ build_udp-discovery-cpp()
B_dir="udp-discovery-cpp-build"
rm -rf ${B_dir}; mkdir ${B_dir}; cd ${B_dir}
cmake \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -H../udp-discovery-cpp
andro_cmake -B. -H../$S_dir
make -j${HOST_NUM_CPU}
cp libudp-discovery.a "${PREFIX}/lib/"
cp ../$S_dir/*.hpp "${PREFIX}/include/"
cd ..
}
task_register build_xapian
build_xapian()
{
echo "build_xapian()
################################################################################
################################################################################
################################################################################
"
B_dir="xapian-core-${XAPIAN_SOURCE_VERSION}"
D_file="$B_dir.tar.xz"
verified_download $D_file $XAPIAN_SOURCE_SHA256 \
@ -503,15 +635,12 @@ build_xapian()
--prefix="${PREFIX}" --with-sysroot="${SYSROOT}"
make -j${HOST_NUM_CPU}
make install
cd ..
}
task_register build_miniupnpc
build_miniupnpc()
{
echo "build_miniupnpc()
################################################################################
################################################################################
################################################################################
"
S_dir="miniupnpc-${MINIUPNPC_SOURCE_VERSION}"
B_dir="miniupnpc-${MINIUPNPC_SOURCE_VERSION}-build"
D_file="$S_dir.tar.gz"
@ -521,29 +650,229 @@ build_miniupnpc()
tar -xf $D_file
mkdir $B_dir
cd $B_dir
cmake \
andro_cmake \
-DUPNPC_BUILD_STATIC=TRUE \
-DUPNPC_BUILD_SHARED=FALSE \
-DUPNPC_BUILD_TESTS=FALSE \
-DUPNPC_BUILD_SAMPLE=FALSE \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -S../$S_dir
-B. -S../$S_dir
make -j${HOST_NUM_CPU}
make install
cd ..
}
build_toolchain
[ "${INSTALL_QT_ANDROID}X" != "trueX" ] || install_qt_android
build_bzlib
build_openssl
build_sqlite
build_sqlcipher
build_libupnp
build_rapidjson
build_restbed
build_udp-discovery-cpp
build_xapian
build_miniupnpc
delete_copied_includes
task_register build_zlib
build_zlib()
{
S_dir="zlib-${ZLIB_SOURCE_VERSION}"
B_dir="zlib-${ZLIB_SOURCE_VERSION}-build"
D_file="$S_dir.tar.xz"
verified_download $D_file $ZLIB_SOURCE_SHA256 \
http://www.zlib.net/${D_file}
rm -rf $S_dir $B_dir
tar -xf $D_file
mkdir $B_dir
cd $B_dir
andro_cmake -B. -S../$S_dir
make -j${HOST_NUM_CPU}
make install
rm -f ${PREFIX}/lib/libz.so*
cd ..
}
echo NATIVE_LIBS_TOOLCHAIN_PATH=${NATIVE_LIBS_TOOLCHAIN_PATH}
task_register build_libpng
build_libpng()
{
task_run build_zlib
S_dir="libpng-${LIBPNG_SOURCE_VERSION}"
B_dir="libpng-${LIBPNG_SOURCE_VERSION}-build"
D_file="$S_dir.tar.xz"
verified_download $D_file $LIBPNG_SOURCE_SHA256 \
https://download.sourceforge.net/libpng/${D_file}
rm -rf $S_dir $B_dir
tar -xf $D_file
# libm is part of bionic An android
sed -i -e 's/find_library(M_LIBRARY m)/set(M_LIBRARY "")/' $S_dir/CMakeLists.txt
# Disable hardware acceleration as they are problematic for Android
# compilation and are not supported by all phones, it is necessary to fiddle
# with CMakeLists.txt as libpng 1.6.37 passing it as cmake options seems not
# working properly
# https://github.com/imagemin/optipng-bin/issues/97
# https://github.com/opencv/opencv/issues/7600
echo "add_definitions(-DPNG_ARM_NEON_OPT=0)" >> $S_dir/CMakeLists.txt
mkdir $B_dir
pushd $B_dir
HW_OPT="OFF"
# [ "$ANDROID_PLATFORM_VER" -ge "22" ] && HW_OPT="ON"
andro_cmake \
-DPNG_SHARED=OFF \
-DPNG_STATIC=ON \
-DPNG_TESTS=OFF \
-DPNG_HARDWARE_OPTIMIZATIONS=$HW_OPT \
-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
-B. -S../$S_dir
make -j${HOST_NUM_CPU}
make install
popd
}
task_register build_libjpeg
build_libjpeg()
{
S_dir="jpeg-${LIBJPEG_SOURCE_VERSION}"
D_file="jpegsrc.v${LIBJPEG_SOURCE_VERSION}.tar.gz"
verified_download $D_file $LIBJPEG_SOURCE_SHA256 \
https://www.ijg.org/files/$D_file
rm -rf $S_dir
tar -xf $D_file
cd $S_dir
./configure --with-pic --prefix="${PREFIX}" --host=${cArch}-linux
make -j${HOST_NUM_CPU}
make install
rm -f ${PREFIX}/lib/libjpeg.so*
cd ..
}
task_register build_tiff
build_tiff()
{
S_dir="tiff-${TIFF_SOURCE_VERSION}"
B_dir="${S_dir}-build"
D_file="tiff-${TIFF_SOURCE_VERSION}.tar.gz"
verified_download $D_file $TIFF_SOURCE_SHA256 \
https://download.osgeo.org/libtiff/${D_file}
rm -rf $S_dir $B_dir
tar -xf $D_file
mkdir $B_dir
# Disable tools building, not needed for retroshare, and depending on some
# OpenGL headers not available on Android
echo "" > $S_dir/tools/CMakeLists.txt
# Disable tests building, not needed for retroshare, and causing linker
# errors
echo "" > $S_dir/test/CMakeLists.txt
# Disable extra tools building, not needed for retroshare, and causing
# linker errors
echo "" > $S_dir/contrib/CMakeLists.txt
# Disable more unneded stuff
echo "" > $S_dir/build/CMakeLists.txt
echo "" > $S_dir/html/CMakeLists.txt
echo "" > $S_dir/man/CMakeLists.txt
echo "" > $S_dir/port/CMakeLists.txt
# Change to static library build
sed -i 's\add_library(tiff\add_library(tiff STATIC\' \
$S_dir/libtiff/CMakeLists.txt
cd $B_dir
#TODO: build dependecies to support more formats
andro_cmake \
-Dlibdeflate=OFF -Djbig=OFF -Dlzma=OFF -Dzstd=OFF -Dwebp=OFF \
-Djpeg12=OFF \
-Dcxx=OFF \
-B. -S../$S_dir
make -j${HOST_NUM_CPU}
make install
cd ..
}
task_register build_cimg
build_cimg()
{
task_run build_libpng
task_run build_libjpeg
task_run build_tiff
S_dir="CImg-${CIMG_SOURCE_VERSION}"
D_file="CImg_${CIMG_SOURCE_VERSION}.zip"
verified_download $D_file $CIMG_SOURCE_SHA256 \
https://cimg.eu/files/${D_file}
unzip -o $D_file
cp --archive --verbose "$S_dir/CImg.h" "$PREFIX/include/"
}
task_register build_phash
build_phash()
{
task_run build_cimg
S_dir="pHash"
B_dir="${S_dir}-build"
git_source_get "$S_dir" "$PHASH_SOURCE_REPO" "${PHASH_SOURCE_VERSION}"
rm -rf $B_dir; mkdir $B_dir ; pushd $B_dir
andro_cmake -DPHASH_DYNAMIC=OFF -DPHASH_STATIC=ON -B. -H../pHash
make -j${HOST_NUM_CPU}
make install
popd
}
task_register build_mvptree
build_mvptree()
{
S_dir="mvptree"
B_dir="${S_dir}-build"
git_source_get "$S_dir" "$MVPTREE_SOURCE_REPO" "${MVPTREE_SOURCE_VERSION}"
rm -rf $B_dir; mkdir $B_dir ; pushd $B_dir
andro_cmake -B. -H../${S_dir}
make -j${HOST_NUM_CPU}
make install
popd
}
task_register get_native_libs_toolchain_path
get_native_libs_toolchain_path()
{
echo ${NATIVE_LIBS_TOOLCHAIN_PATH}
}
task_register build_default_toolchain
build_default_toolchain()
{
task_run bootstrap_toolchain || return $?
task_run build_bzlib || return $?
task_run build_openssl || return $?
task_run build_sqlcipher || return $?
task_run build_rapidjson || return $?
task_run build_restbed || return $?
task_run build_udp-discovery-cpp || return $?
task_run build_xapian || return $?
task_run build_miniupnpc || return $?
task_run build_phash || return $?
task_run build_mvptree || return $?
task_run deduplicate_includes || return $?
task_run get_native_libs_toolchain_path || return $?
}
if [ "$1" == "" ]; then
rm -rf "$REPORT_DIR"
mkdir -p "$REPORT_DIR"
cat "$0" > "$REPORT_DIR/build_script"
env > "$REPORT_DIR/build_env"
build_default_toolchain
else
# do not delete report directory in this case so we can reuse material
# produced by previous run, like deduplicated includes
mkdir -p "$REPORT_DIR"
while [ "$1" != "" ] ; do
task_zap $1
task_run $1 || exit $?
shift
done
fi

View File

@ -0,0 +1,43 @@
ARG ANDROID_NDK_ARCH=arm
FROM registry.gitlab.com/retroshare/retroshare:android_${ANDROID_NDK_ARCH}_base
RUN apt-get update -y && apt-get upgrade -y
ARG REPO_URL=https://gitlab.com/RetroShare/RetroShare.git
ARG REPO_BRANCH=master
WORKDIR /RetroShare
RUN git remote add testing $REPO_URL && \
git fetch --tags testing $REPO_BRANCH && \
git reset --hard testing/$REPO_BRANCH && \
git --no-pager log --max-count 1
RUN rm -rf /jsonapi-generator-build ; mkdir /jsonapi-generator-build
WORKDIR /jsonapi-generator-build/
RUN qmake ../RetroShare/jsonapi-generator/src/ \
CONFIG+=no_retroshare_plugins \
CONFIG+=no_retroshare_service CONFIG+=no_retroshare_gui \
CONFIG+=rs_jsonapi && \
make -j$(nproc)
RUN rm -rf /retroshare-service-android-build ; mkdir /retroshare-service-android-build
WORKDIR /retroshare-service-android-build
# ARG declared before FROM get wiped after it, so we need declaring it again
ARG ANDROID_NDK_ARCH=arm
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/qmake ../RetroShare \
-spec android-clang \
CONFIG+=retroshare_service CONFIG+=rs_jsonapi \
RS_UPNP_LIB=miniupnpc \
JSONAPI_GENERATOR_EXE=/jsonapi-generator-build/jsonapi-generator \
NATIVE_LIBS_TOOLCHAIN_PATH=$NATIVE_LIBS_TOOLCHAIN_PATH \
CONFIG+=no_retroshare_gui CONFIG+=no_rs_service_webui_terminal_password \
CONFIG+=no_rs_service_terminal_login
RUN make -j$(nproc)
RUN make install INSTALL_ROOT=/retroshare-service-android-build/android-build/
ARG ANDROID_PLATFORM_VER=16
RUN $($PREPARE_TOOLCHAIN get_qt_dir | head -n 1)/bin/androiddeployqt \
--input retroshare-service/src/android-libretroshare-service.so-deployment-settings.json \
--output android-build --android-platform android-$ANDROID_PLATFORM_VER \
--jdk $JAVA_HOME --gradle