From b3bef085f850ff0710923bf5ed923212cfa523c4 Mon Sep 17 00:00:00 2001 From: steadfasterX Date: Thu, 30 Mar 2023 13:32:30 +0000 Subject: [PATCH] allow to specify thread counts (sync,build) when DOS_MAX_THREADS_REPO and/or DOS_MAX_THREADS_BUILD are defined in init.sh they will be used. otherwise the max threads are calculated based on the cpu cores count. if that calculation fails (depends on nproc) a default fallback of: - 4 is used for repo sync (which reflects the current state for repo sync) - `` is used for make - which yields to no limitation (which reflects the current state for mka / make) to avoid issues regarding being rate limited a max value for repo sync processes will overwrite DOS_MAX_THREADS_REPO when set greater than MAX_THREADS_REPO_RATE (8). Signed-off-by: steadfasterX --- Scripts/Common/Functions.sh | 4 +-- Scripts/LineageOS-14.1/Functions.sh | 4 +-- Scripts/LineageOS-15.1/Functions.sh | 2 +- Scripts/LineageOS-16.0/Functions.sh | 2 +- Scripts/LineageOS-17.1/Functions.sh | 2 +- Scripts/LineageOS-18.1/Functions.sh | 2 +- Scripts/LineageOS-19.1/Functions.sh | 2 +- Scripts/LineageOS-20.0/Functions.sh | 2 +- Scripts/init.sh | 41 +++++++++++++++++++++++++++++ 9 files changed, 51 insertions(+), 10 deletions(-) diff --git a/Scripts/Common/Functions.sh b/Scripts/Common/Functions.sh index 327d0ecb..47e12ea5 100644 --- a/Scripts/Common/Functions.sh +++ b/Scripts/Common/Functions.sh @@ -23,7 +23,7 @@ export -f startPatcher; resetWorkspace() { umask 0022; - repo forall -c 'git add -A && git reset --hard' && rm -rf out DOS_PATCHED_FLAG && repo sync -j8 --force-sync --detach; + repo forall -j${DOS_MAX_THREADS_REPO} -c 'git add -A && git reset --hard' && rm -rf out DOS_PATCHED_FLAG && repo sync -j${DOS_MAX_THREADS_REPO} --force-sync --detach; } export -f resetWorkspace; @@ -33,7 +33,7 @@ patchWorkspace() { export -f patchWorkspace; verifyAllPlatformTags() { - repo forall -v -c 'sh -c "source $DOS_WORKSPACE_ROOT/Scripts/Common/Tag_Verifier.sh && verifyTagIfPlatform $REPO_PROJECT $REPO_PATH"'; + repo forall -j${DOS_MAX_THREADS_REPO} -v -c 'sh -c "source $DOS_WORKSPACE_ROOT/Scripts/Common/Tag_Verifier.sh && verifyTagIfPlatform $REPO_PROJECT $REPO_PATH"'; } export -f verifyAllPlatformTags; diff --git a/Scripts/LineageOS-14.1/Functions.sh b/Scripts/LineageOS-14.1/Functions.sh index 67a33118..055ab050 100644 --- a/Scripts/LineageOS-14.1/Functions.sh +++ b/Scripts/LineageOS-14.1/Functions.sh @@ -34,7 +34,7 @@ buildDevice() { cd "$DOS_BUILD_BASE"; export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1"; pkill java && sleep 10; #XXX: ugly hack - breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2; + breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2; pkill java && sleep 10; #XXX: ugly hack } export -f buildDevice; @@ -43,7 +43,7 @@ buildDeviceUserDebug() { cd "$DOS_BUILD_BASE"; if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1"; - breakfast "lineage_$1-userdebug" && mka target-files-package otatools && processRelease $1 true $2; + breakfast "lineage_$1-userdebug" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2; else echo -e "\e[0;31mNo signing keys available for $1\e[0m"; fi; diff --git a/Scripts/LineageOS-15.1/Functions.sh b/Scripts/LineageOS-15.1/Functions.sh index 0139637f..4732a301 100644 --- a/Scripts/LineageOS-15.1/Functions.sh +++ b/Scripts/LineageOS-15.1/Functions.sh @@ -34,7 +34,7 @@ buildDevice() { cd "$DOS_BUILD_BASE"; if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1"; - breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2; + breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2; else echo -e "\e[0;31mNo signing keys available for $1\e[0m"; fi; diff --git a/Scripts/LineageOS-16.0/Functions.sh b/Scripts/LineageOS-16.0/Functions.sh index 7a815dec..df83ddc5 100644 --- a/Scripts/LineageOS-16.0/Functions.sh +++ b/Scripts/LineageOS-16.0/Functions.sh @@ -34,7 +34,7 @@ buildDevice() { cd "$DOS_BUILD_BASE"; if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1"; - breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2; + breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2; else echo -e "\e[0;31mNo signing keys available for $1\e[0m"; fi; diff --git a/Scripts/LineageOS-17.1/Functions.sh b/Scripts/LineageOS-17.1/Functions.sh index 5e28f645..f876f41b 100644 --- a/Scripts/LineageOS-17.1/Functions.sh +++ b/Scripts/LineageOS-17.1/Functions.sh @@ -34,7 +34,7 @@ buildDevice() { cd "$DOS_BUILD_BASE"; if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1"; - breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2; + breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2; else echo -e "\e[0;31mNo signing keys available for $1\e[0m"; fi; diff --git a/Scripts/LineageOS-18.1/Functions.sh b/Scripts/LineageOS-18.1/Functions.sh index d1aebc7c..d4143f45 100644 --- a/Scripts/LineageOS-18.1/Functions.sh +++ b/Scripts/LineageOS-18.1/Functions.sh @@ -34,7 +34,7 @@ buildDevice() { cd "$DOS_BUILD_BASE"; if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1"; - breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2; + breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2; else echo -e "\e[0;31mNo signing keys available for $1\e[0m"; fi; diff --git a/Scripts/LineageOS-19.1/Functions.sh b/Scripts/LineageOS-19.1/Functions.sh index a94d2436..cb61d822 100644 --- a/Scripts/LineageOS-19.1/Functions.sh +++ b/Scripts/LineageOS-19.1/Functions.sh @@ -34,7 +34,7 @@ buildDevice() { cd "$DOS_BUILD_BASE"; if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1"; - breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2; + breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2; else echo -e "\e[0;31mNo signing keys available for $1\e[0m"; fi; diff --git a/Scripts/LineageOS-20.0/Functions.sh b/Scripts/LineageOS-20.0/Functions.sh index 1ea238e2..48c27f6e 100644 --- a/Scripts/LineageOS-20.0/Functions.sh +++ b/Scripts/LineageOS-20.0/Functions.sh @@ -34,7 +34,7 @@ buildDevice() { cd "$DOS_BUILD_BASE"; if [[ -d "$DOS_SIGNING_KEYS/$1" ]]; then #export OTA_KEY_OVERRIDE_DIR="$DOS_SIGNING_KEYS/$1"; - breakfast "lineage_$1-user" && mka target-files-package otatools && processRelease $1 true $2; + breakfast "lineage_$1-user" && mka -j${DOS_MAX_THREADS_BUILD} target-files-package otatools && processRelease $1 true $2; else echo -e "\e[0;31mNo signing keys available for $1\e[0m"; fi; diff --git a/Scripts/init.sh b/Scripts/init.sh index 74552274..8b7a2c39 100644 --- a/Scripts/init.sh +++ b/Scripts/init.sh @@ -21,6 +21,8 @@ #START OF USER CONFIGURABLE OPTIONS # #General +#export DOS_MAX_THREADS_REPO=8; #Max amount of threads used for sync, e.g. used by repo (shouldn't exceed your CPU cores count, due to rate limit restrictions this cannot be set higher then 8) +#export DOS_MAX_THREADS_BUILD="nolimit"; #Max amount of threads used for build, e.g. used by make (shouldn't exceed your CPU cores count) export DOS_WORKSPACE_ROOT="/mnt/dos/"; #XXX: THIS MUST BE CORRECT TO BUILD! #export DOS_BUILDS=$DOS_WORKSPACE_ROOT"Builds/"; export DOS_BUILDS="/mnt/Drive-4/DOS/Builds/"; #XXX: THIS MUST BE CORRECT TO BUILD! @@ -109,6 +111,45 @@ export DOS_THEME_700="E64A19"; #Deep Orange 700 umask 0022; +# by default we will calculate the max CPU count automatically (used by e.g. repo commands) +# if the required tool "nproc" is not found a default is used. +export FALLBACK_MAX_THREADS_REPO=4; +export MAX_THREADS_REPO_RATE=8; #to avoid being rate limited we never go above this for syncing +export FALLBACK_MAX_THREADS_BUILD="nolimit"; #if nothing specified we will use all CPU power available + +calcThreads(){ + unset _MAX_THREADS + DEFAULT_COUNT=$1 + nproc --version > /dev/null 2>&1 + if [ $? -eq 10 ];then + _MAX_THREADS_ALL=$(nproc --all) + _MAX_THREADS=$(( _MAX_THREADS_ALL - 1)) + fi + if [ -z "$_MAX_THREADS" ];then + echo $DEFAULT_COUNT + return 9 + else + echo $_MAX_THREADS + fi +} +export -f calcThreads; + +if [ -z "$DOS_MAX_THREADS_REPO" ];then + export DOS_MAX_THREADS_REPO=$(calcThreads $FALLBACK_MAX_THREADS_REPO) \ + || echo -e "\e[0;33mWARNING: could not calculate thread count and no user defined amount specified. This could slow down the SYNC processes.\e[0m" +fi +if [ "$DOS_MAX_THREADS_REPO" -gt "$MAX_THREADS_REPO_RATE" ];then + echo -e "\e[0;33mWARNING: Your specified DOS_MAX_THREADS_REPO value ($DOS_MAX_THREADS_REPO) could raise rate limits so has been decreased to >$MAX_THREADS_REPO_RATE<.\e[0m" + export DOS_MAX_THREADS_REPO=$MAX_THREADS_REPO_RATE +fi +if [ -z "$DOS_MAX_THREADS_BUILD" ];then + export DOS_MAX_THREADS_BUILD=$(calcThreads $FALLBACK_MAX_THREADS_BUILD)\ + || echo -e "\e[0;33mWARNING: could not calculate thread count and no user defined amount specified. This could slow down the BUILD processes.\e[0m" +fi +echo "Activated CPU count (sync): $DOS_MAX_THREADS_REPO" +echo "Activated CPU count (build): $DOS_MAX_THREADS_BUILD" +[ "$DOS_MAX_THREADS_BUILD" == "nolimit" ] && export DOS_MAX_THREADS_BUILD="" #actually means "all" when empty + gpgVerifyGitHead() { if [ -r "$DOS_TMP_GNUPG/pubring.kbx" ]; then if git -C "$1" verify-commit HEAD &>/dev/null; then