From e10a865b05ceb4a990b27e44edb5bd872c96acc2 Mon Sep 17 00:00:00 2001 From: Tad Date: Thu, 29 Aug 2019 14:37:06 -0400 Subject: [PATCH] Improve release processing to support deltas and archiving --- .../0001-Automated_Build_Signing.patch | 103 ------------------ Scripts/Common/Functions.sh | 65 ++++++++--- Scripts/LineageOS-11.0/Functions.sh | 4 +- Scripts/LineageOS-11.0/Patch.sh | 3 - Scripts/LineageOS-14.1/Functions.sh | 99 +++++++++-------- Scripts/LineageOS-14.1/Patch.sh | 4 - Scripts/LineageOS-15.1/Functions.sh | 2 +- Scripts/LineageOS-15.1/Patch.sh | 3 - Scripts/LineageOS-16.0/Functions.sh | 2 +- Scripts/LineageOS-16.0/Patch.sh | 3 - Scripts/init.sh | 4 + TODO | 3 +- 12 files changed, 111 insertions(+), 184 deletions(-) delete mode 100644 Patches/LineageOS-14.1/android_build/0001-Automated_Build_Signing.patch diff --git a/Patches/LineageOS-14.1/android_build/0001-Automated_Build_Signing.patch b/Patches/LineageOS-14.1/android_build/0001-Automated_Build_Signing.patch deleted file mode 100644 index 19e00f58..00000000 --- a/Patches/LineageOS-14.1/android_build/0001-Automated_Build_Signing.patch +++ /dev/null @@ -1,103 +0,0 @@ -From 6b6a130924ba6d881d2db44d6382ab96fcb061d9 Mon Sep 17 00:00:00 2001 -From: Daniel Micay -Date: Tue, 3 Apr 2018 12:32:43 -0400 -Subject: [PATCH] Add optional automated signing - -Change-Id: I2a7efac25b3e48d3edcadcb9a57e9a7dbc7dfac0 ---- - core/Makefile | 43 +++++++++++++++++++++++++++++++++++++------ - 1 file changed, 37 insertions(+), 6 deletions(-) - -diff --git a/core/Makefile b/core/Makefile -index 3fb424733..207088b93 100644 ---- a/core/Makefile -+++ b/core/Makefile -@@ -416,6 +416,11 @@ ifneq ($(OTA_PACKAGE_SIGNING_KEY),) - DEFAULT_KEY_CERT_PAIR := $(OTA_PACKAGE_SIGNING_KEY) - endif - -+ifneq ($(SIGNING_KEY_DIR),) -+ KEY_CERT_DIR := $(SIGNING_KEY_DIR) -+ DEFAULT_KEY_CERT_PAIR := $(SIGNING_KEY_DIR)/releasekey -+endif -+ - # Rules that need to be present for the all targets, even - # if they don't do anything. - .PHONY: systemimage -@@ -1033,11 +1038,15 @@ endif - # substitute other keys for this one. - OTA_PUBLIC_KEYS := $(DEFAULT_SYSTEM_DEV_CERTIFICATE).x509.pem - --ifneq ($(OTA_PACKAGE_SIGNING_KEY),) -- OTA_PUBLIC_KEYS := $(OTA_PACKAGE_SIGNING_KEY).x509.pem -- PRODUCT_EXTRA_RECOVERY_KEYS := $(DEFAULT_SYSTEM_DEV_CERTIFICATE) -+ifneq ($(SIGNING_KEY_DIR),) -+ OTA_PUBLIC_KEYS := $(SIGNING_KEY_DIR)/releasekey.x509.pem -+ PRODUCT_EXTRA_RECOVERY_KEYS += $(SIGNING_KEY_DIR)/extra -+else -+ ifneq ($(OTA_PACKAGE_SIGNING_KEY),) -+ OTA_PUBLIC_KEYS := $(OTA_PACKAGE_SIGNING_KEY).x509.pem -+ PRODUCT_EXTRA_RECOVERY_KEYS := $(DEFAULT_SYSTEM_DEV_CERTIFICATE) -+ endif - endif -- - # Generate a file containing the keys that will be read by the - # recovery binary. - RECOVERY_INSTALL_OTA_KEYS := \ -@@ -1823,6 +1832,13 @@ $(BUILT_TARGET_FILES_PACKAGE): intermediates := $(intermediates) - $(BUILT_TARGET_FILES_PACKAGE): \ - zip_root := $(intermediates)/$(name) - -+SIGNED_TARGET_FILES_PACKAGE := $(intermediates)/signed-$(name).zip -+MAYBE_SIGNED_TARGET_FILES_PACKAGE := $(BUILT_TARGET_FILES_PACKAGE) -+ -+ifneq ($(SIGNING_KEY_DIR),) -+ MAYBE_SIGNED_TARGET_FILES_PACKAGE := $(SIGNED_TARGET_FILES_PACKAGE) -+endif -+ - # $(1): Directory to copy - # $(2): Location to copy it to - # The "ls -A" is to prevent "acp s/* d" from failing if s is empty. -@@ -2190,6 +2206,12 @@ else - OTA_FROM_TARGET_SCRIPT := $(TARGET_RELEASETOOL_OTA_FROM_TARGET_SCRIPT) - endif - -+ifeq ($(TARGET_RELEASETOOL_SIGN_TARGET_SCRIPT),) -+ SIGN_TARGET_SCRIPT := ./build/tools/releasetools/sign_target_files_apks -+else -+ SIGN_TARGET_SCRIPT := $(TARGET_RELEASETOOL_SIGN_TARGET_SCRIPT) -+endif -+ - ifeq ($(WITH_GMS),true) - $(INTERNAL_OTA_PACKAGE_TARGET): backuptool := false - else -@@ -2210,7 +2232,16 @@ ifneq ($(BLOCK_BASED_OTA),false) - $(INTERNAL_OTA_PACKAGE_TARGET): block_based := --block - endif - --$(INTERNAL_OTA_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE) -+$(SIGNED_TARGET_FILES_PACKAGE): $(BUILT_TARGET_FILES_PACKAGE) -+ @echo "$(SIGN_TARGET_SCRIPT)" > $(PRODUCT_OUT)/sign_script_path -+ @echo -e ${CL_YLW}"Sign target files:"${CL_RST}" $@" -+ $(hide) $(SIGN_TARGET_SCRIPT) \ -+ -d $(KEY_CERT_DIR) \ -+ -o \ -+ $(BUILT_TARGET_FILES_PACKAGE) \ -+ $(SIGNED_TARGET_FILES_PACKAGE) -+ -+$(INTERNAL_OTA_PACKAGE_TARGET): $(MAYBE_SIGNED_TARGET_FILES_PACKAGE) - @echo "$(OTA_FROM_TARGET_SCRIPT)" > $(PRODUCT_OUT)/ota_script_path - @echo "Package OTA: $@" - $(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH MKBOOTIMG=$(MKBOOTIMG) \ -@@ -2220,7 +2251,7 @@ $(INTERNAL_OTA_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE) - -k $(KEY_CERT_PAIR) \ - --backup=$(backuptool) \ - $(if $(OEM_OTA_CONFIG), -o $(OEM_OTA_CONFIG)) \ -- $(BUILT_TARGET_FILES_PACKAGE) $@ -+ $(MAYBE_SIGNED_TARGET_FILES_PACKAGE) $@ - - CM_TARGET_PACKAGE := $(PRODUCT_OUT)/lineage-$(LINEAGE_VERSION).zip - --- -2.16.3 - diff --git a/Scripts/Common/Functions.sh b/Scripts/Common/Functions.sh index c7df95b6..fce79df7 100644 --- a/Scripts/Common/Functions.sh +++ b/Scripts/Common/Functions.sh @@ -140,38 +140,73 @@ audit2allowADB() { } export -f audit2allowADB; -signRelease() { +processRelease() { #https://github.com/GrapheneOS/script/blob/pie/release.sh - DEVICE=$1; - VERITY=$2; + DEVICE="$1"; + BLOCK="$2"; + VERITY="$3"; DATE=$(date -u '+%Y%m%d') - KEY_DIR=$DOS_SIGNING_KEYS; - PREFIX="lineage_"; + KEY_DIR="$DOS_SIGNING_KEYS"; VERSION=$(echo $DOS_VERSION | cut -f2 -d "-"); - TARGET_FILES=divested-$VERSION-$DATE-dos-$DEVICE-target_files.zip; + PREFIX="$DOS_BRANDING_ZIP_PREFIX-$VERSION-$DATE-dos-$DEVICE"; + ARCHIVE="$DOS_BUILDS/$DOS_VERSION/release_keys/"; + OUT_DIR="$DOS_BUILD_BASE/out/target/product/$DEVICE/"; + #Arguments + if [ "$BLOCK" != false ]; then + BLOCK_SWITCHES="--block"; + fi; if [ "$VERITY" = true ]; then VERITY_SWITCHES=(--replace_verity_public_key "$KEY_DIR/verity_key.pub" \ --replace_verity_private_key "$KEY_DIR/verity" \ --replace_verity_keyid "$KEY_DIR/verity.x509.pem"); fi; + #Target Files build/tools/releasetools/sign_target_files_apks -o -d "$KEY_DIR" \ "${VERITY_SWITCHES[@]}" \ - out/target/product/$DEVICE/obj/PACKAGING/target_files_intermediates/$PREFIX$DEVICE-target_files-*.zip \ - $OUT/$TARGET_FILES; + $OUT_DIR/obj/PACKAGING/target_files_intermediates/*$DEVICE-target_files-*.zip \ + $OUT_DIR/$PREFIX-target_files.zip; + INCREMENTAL_ID=$(grep "ro.build.version.incremental" $OUT_DIR/system/build.prop | cut -f2 -d "=" | sed 's/\.//g'); + echo $INCREMENTAL_ID > $OUT_DIR/$PREFIX-target_files.zip.id; - build/tools/releasetools/ota_from_target_files --block -k "$KEY_DIR/releasekey" \ - $OUT/$TARGET_FILES \ - $OUT/divested-$VERSION-$DATE-dos-$DEVICE-ota.zip; + #Image + #build/tools/releasetools/img_from_target_files $OUT_DIR/$PREFIX-target_files.zip \ + # $OUT_DIR/$PREFIX-img.zip || exit 1; - md5sum $OUT/divested-$VERSION-$DATE-dos-$DEVICE-ota.zip > $OUT/divested-$VERSION-$DATE-dos-$DEVICE-ota.zip.md5sum; + #OTA + build/tools/releasetools/ota_from_target_files $BLOCK_SWITCHES -t 8 -k "$KEY_DIR/releasekey" \ + $OUT_DIR/$PREFIX-target_files.zip \ + $OUT_DIR/$PREFIX-ota.zip; + md5sum $OUT_DIR/$PREFIX-ota.zip > $OUT_DIR/$PREFIX-ota.zip.md5sum; - #build/tools/releasetools/img_from_target_files $OUT/$TARGET_FILES \ - # $OUT/divested-$VERSION-$DATE-dos-$DEVICE-img.zip || exit 1; + #Deltas + if [ "$DOS_GENERATE_DELTAS" = true ]; then + for LAST_TARGET_FILES in $ARCHIVE/target_files/$DOS_BRANDING_ZIP_PREFIX-$VERSION-*-dos-$DEVICE-target_files.zip; do + if [[ -f "$LAST_TARGET_FILES.id" ]]; then + LAST_INCREMENTAL_ID=$(cat "$LAST_TARGET_FILES.id"); + build/tools/releasetools/ota_from_target_files $BLOCK_SWITCHES -t 8 -k "$KEY_DIR" -i \ + "$LAST_TARGET_FILES" \ + $OUT_DIR/$PREFIX-target_files.zip \ + $OUT_DIR/$PREFIX-incremental_$LAST_INCREMENTAL_ID.zip; + md5sum $OUT_DIR/$PREFIX-incremental_$LAST_INCREMENTAL_ID.zip > $OUT_DIR/$PREFIX-incremental_$LAST_INCREMENTAL_ID.zip.md5sum; + fi; + done; + fi; + + #Copy to archive + if [ "$DOS_AUTO_ARCHIVE_BUILDS" = true ]; then + mkdir -vp $ARCHIVE; + mkdir -vp $ARCHIVE/target_files; + mkdir -vp $ARCHIVE/incrementals; + + cp -v $OUT_DIR/$PREFIX-target_files.* $ARCHIVE/target_files/; + cp -v $OUT_DIR/$PREFIX-ota.zip* $ARCHIVE/; + cp -v $OUT_DIR/$PREFIX-incremental_*.zip* $ARCHIVE/incrementals/; + fi; } -export -f signRelease; +export -f processRelease; disableDexPreOpt() { cd "$DOS_BUILD_BASE$1"; diff --git a/Scripts/LineageOS-11.0/Functions.sh b/Scripts/LineageOS-11.0/Functions.sh index c0b81b36..82e06961 100644 --- a/Scripts/LineageOS-11.0/Functions.sh +++ b/Scripts/LineageOS-11.0/Functions.sh @@ -35,7 +35,7 @@ scanWorkspaceForMalware() { export -f scanWorkspaceForMalware; buildDevice() { - brunch "cm_$1-user"; + brunch "cm_$1-user" && processRelease $1 false false; } export -f buildDevice; @@ -48,7 +48,7 @@ export -f buildDeviceDebug; buildAll() { if [ "$DOS_MALWARE_SCAN_ENABLED" = true ]; then scanWorkspaceForMalware; fi; if [ "$DOS_OPTIMIZE_IMAGES" = true ]; then optimizeImagesRecursive "$DOS_BUILD_BASE"; fi; - brunch lineage_nex-user; + brunch lineage_nex-user && processRelease nex false false; } export -f buildAll; diff --git a/Scripts/LineageOS-11.0/Patch.sh b/Scripts/LineageOS-11.0/Patch.sh index c720dfad..c1db4abf 100644 --- a/Scripts/LineageOS-11.0/Patch.sh +++ b/Scripts/LineageOS-11.0/Patch.sh @@ -30,9 +30,6 @@ #buildDevice [device] #buildAll -#Generate an incremental -#./build/tools/releasetools/ota_from_target_files --block -t 8 -i old.zip new.zip update.zip - #Generate firmware deblobber #mka firmware_deblobber diff --git a/Scripts/LineageOS-14.1/Functions.sh b/Scripts/LineageOS-14.1/Functions.sh index cef19850..dacf3b70 100644 --- a/Scripts/LineageOS-14.1/Functions.sh +++ b/Scripts/LineageOS-14.1/Functions.sh @@ -35,10 +35,15 @@ scanWorkspaceForMalware() { export -f scanWorkspaceForMalware; buildDevice() { - brunch "lineage_$1-user"; + brunch "lineage_$1-user" && processRelease $1 true $2; } export -f buildDevice; +buildDeviceUserDebug() { + brunch "lineage_$1-userdebug" && processRelease $1 true $2; +} +export -f buildDeviceUserDebug; + buildDeviceDebug() { unset SIGNING_KEY_DIR; brunch "lineage_$1-eng"; @@ -49,55 +54,55 @@ buildAll() { if [ "$DOS_MALWARE_SCAN_ENABLED" = true ]; then scanWorkspaceForMalware; fi; if [ "$DOS_OPTIMIZE_IMAGES" = true ]; then optimizeImagesRecursive "$DOS_BUILD_BASE"; fi; #Select devices are userdebug due to SELinux policy issues - brunch lineage_clark-user; - brunch lineage_thor-userdebug; - brunch lineage_grouper-user; #needs manual patching - one-repo vendor blob patch - brunch lineage_h815-user; - brunch lineage_herolte-user; - brunch lineage_himaul-user; - brunch lineage_i9100-userdebug; - brunch lineage_i9300-userdebug; - brunch lineage_i9305-user; - brunch lineage_maguro-user; - brunch lineage_manta-user; - brunch lineage_n5110-user; - #brunch lineage_n7100-user; #device/samsung/n7100/selinux/device.te:5:ERROR 'duplicate declaration of type/attribute' at token ';': type hpd_device, dev_type; type mfc_device, dev_type; - brunch lineage_osprey-user; - brunch lineage_toro-user; - brunch lineage_toroplus-user; - brunch lineage_Z00T-user; + buildDevice clark; + buildDeviceUserDebug thor; + buildDevice grouper; #needs manual patching - one-repo vendor blob patch + buildDevice h815; + buildDevice herolte; + buildDevice himaul; + buildDeviceUserDebug i9100; + buildDeviceUserDebug i9300; + buildDevice i9305; + buildDevice maguro; + buildDevice manta; + buildDevice n5110; + buildDevice n7100; #device/samsung/n7100/selinux/device.te:5:ERROR 'duplicate declaration of type/attribute' at token ';': type hpd_device, dev_type; type mfc_device, dev_type; + buildDevice osprey; + buildDevice toro; + buildDevice toroplus; + buildDevice Z00T; #The following are all superseded, and should only be enabled if the newer version is broken (not building/booting/etc.) if [ "$DOS_BUILDALL_SUPERSEDED" = true ]; then - brunch lineage_angler-user; - brunch lineage_axon7-user; - brunch lineage_bullhead-user; - brunch lineage_bacon-user; - brunch lineage_crackling-user; - brunch lineage_d802-user; - brunch lineage_d852-user; - brunch lineage_d855-user; - brunch lineage_dragon-user; - brunch lineage_ether-user; - brunch lineage_flo-user; - brunch lineage_flounder-user; - brunch lineage_FP2-user; - brunch lineage_fugu-user; - brunch lineage_griffin-user; - brunch lineage_h850-user; - brunch lineage_ham-user; - brunch lineage_hammerhead-user; - brunch lineage_jfltexx-user; #broken - drivers/video/msm/mdp.c:401:1: warning: the frame size of 1032 bytes is larger than 1024 bytes [-Wframe-larger-than=] - brunch lineage_kipper-user; - brunch lineage_klte-user; - brunch lineage_m8-user; - brunch lineage_mako-user; - brunch lineage_marlin-user; - brunch lineage_sailfish-user; - brunch lineage_shamu-user; - brunch lineage_us996-user; - brunch lineage_us997-user; - brunch lineage_victara-user; #needs manual patching - fwb xml: fused: dangling tag + buildDevice angler; + buildDevice axon7; + buildDevice bullhead; + buildDevice bacon; + buildDevice crackling; + buildDevice d802; + buildDevice d852; + buildDevice d855; + buildDevice dragon; + buildDevice ether; + buildDevice flo; + buildDevice flounder; + buildDevice FP2; + buildDevice fugu; + buildDevice griffin; + buildDevice h850; + buildDevice ham; + buildDevice hammerhead; + buildDevice jfltexx; #broken - drivers/video/msm/mdp.c:401:1: warning: the frame size of 1032 bytes is larger than 1024 bytes [-Wframe-larger-than=] + buildDevice kipper; + buildDevice klte; + buildDevice m8; + buildDevice mako; + buildDevice marlin; + buildDevice sailfish; + buildDevice shamu; + buildDevice us996; + buildDevice us997; + buildDevice victara; #needs manual patching - fwb xml: fused: dangling tag fi; } export -f buildAll; diff --git a/Scripts/LineageOS-14.1/Patch.sh b/Scripts/LineageOS-14.1/Patch.sh index 6cb94e28..fe1f452f 100644 --- a/Scripts/LineageOS-14.1/Patch.sh +++ b/Scripts/LineageOS-14.1/Patch.sh @@ -30,9 +30,6 @@ #buildDevice [device] #buildAll -#Generate an incremental -#./build/tools/releasetools/ota_from_target_files --block -t 8 -i old.zip new.zip update.zip - #Generate firmware deblobber #mka firmware_deblobber @@ -69,7 +66,6 @@ enterAndClear "bootable/recovery"; patch -p1 < "$DOS_PATCHES/android_bootable_recovery/0001-Squash_Menus.patch"; #What's a back button? enterAndClear "build"; -patch -p1 < "$DOS_PATCHES/android_build/0001-Automated_Build_Signing.patch"; #Automated build signing (CopperheadOS-13.0) sed -i '50i$(my_res_package): PRIVATE_AAPT_FLAGS += --auto-add-overlay' core/aapt2.mk; sed -i '296iLOCAL_AAPT_FLAGS += --auto-add-overlay' core/package_internal.mk; diff --git a/Scripts/LineageOS-15.1/Functions.sh b/Scripts/LineageOS-15.1/Functions.sh index a178c381..9960dd85 100644 --- a/Scripts/LineageOS-15.1/Functions.sh +++ b/Scripts/LineageOS-15.1/Functions.sh @@ -35,7 +35,7 @@ scanWorkspaceForMalware() { export -f scanWorkspaceForMalware; buildDevice() { - brunch "lineage_$1-user" && signRelease $1 $2; + brunch "lineage_$1-user" && processRelease $1 true $2; } export -f buildDevice; diff --git a/Scripts/LineageOS-15.1/Patch.sh b/Scripts/LineageOS-15.1/Patch.sh index 782c594e..8f6dd05f 100644 --- a/Scripts/LineageOS-15.1/Patch.sh +++ b/Scripts/LineageOS-15.1/Patch.sh @@ -30,9 +30,6 @@ #buildDevice [device] #buildAll -#Generate an incremental -#./build/tools/releasetools/ota_from_target_files --block -t 8 -i old.zip new.zip update.zip - #Generate firmware deblobber #mka firmware_deblobber diff --git a/Scripts/LineageOS-16.0/Functions.sh b/Scripts/LineageOS-16.0/Functions.sh index aa4003fd..359d9a48 100644 --- a/Scripts/LineageOS-16.0/Functions.sh +++ b/Scripts/LineageOS-16.0/Functions.sh @@ -35,7 +35,7 @@ scanWorkspaceForMalware() { export -f scanWorkspaceForMalware; buildDevice() { - brunch "lineage_$1-user" && signRelease $1 $2; + brunch "lineage_$1-user" && processRelease $1 true $2; } export -f buildDevice; diff --git a/Scripts/LineageOS-16.0/Patch.sh b/Scripts/LineageOS-16.0/Patch.sh index 2e9e5f39..5e0559ba 100644 --- a/Scripts/LineageOS-16.0/Patch.sh +++ b/Scripts/LineageOS-16.0/Patch.sh @@ -30,9 +30,6 @@ #buildDevice [device] #buildAll -#Generate an incremental -#./build/tools/releasetools/ota_from_target_files --block -t 8 -i old.zip new.zip update.zip - #Generate firmware deblobber #mka firmware_deblobber diff --git a/Scripts/init.sh b/Scripts/init.sh index d7f0df0c..2494daaf 100644 --- a/Scripts/init.sh +++ b/Scripts/init.sh @@ -53,6 +53,9 @@ export DOS_DEFAULT_DNS_PRESET="OpenNIC"; #Sets default DNS. Options: CensurfriDN export DOS_GPS_NTP_SERVER="1.android.pool.ntp.org"; #Options: Any NTP pool export DOS_GPS_SUPL_HOST="supl.google.com"; #Options: supl.{google,vodafone,sonyericsson}.com +export DOS_AUTO_ARCHIVE_BUILDS=true; #Copies files to $DOS_BUILDS after signing +export DOS_GENERATE_DELTAS=true; #Creates deltas from existing target_files in $DOS_BUILDS + export DOS_BRANDING_NAME="DivestOS"; export DOS_BRANDING_ZIP_PREFIX="divested"; export DOS_BRANDING_BOOTANIMATION_FONT="Fira-Sans-Bold"; #Options: $ convert -list font @@ -111,6 +114,7 @@ export DOS_TMP_DIR="/tmp/dos_tmp"; mkdir -p "$DOS_TMP_DIR"; export DOS_HOSTS_FILE="$DOS_TMP_DIR/hosts"; +export DOS_BUILDS=$DOS_WORKSPACE_ROOT"Builds/"; export DOS_PREBUILT_APPS=$DOS_WORKSPACE_ROOT"PrebuiltApps/"; export DOS_PATCHES_COMMON=$DOS_WORKSPACE_ROOT"Patches/Common/"; export DOS_PATCHES=$DOS_WORKSPACE_ROOT"Patches/$BUILD_WORKING_DIR/"; diff --git a/TODO b/TODO index 428d387a..26d00a13 100644 --- a/TODO +++ b/TODO @@ -1,8 +1,7 @@ -Last updated: 2019-02-08 +Last updated: 2019-08-29 High Priority (Release blockers) Build - - Automate incremental update generation - New signing keys Project - GPG all the things!