Improve release processing to support deltas and archiving

This commit is contained in:
Tad 2019-08-29 14:37:06 -04:00
parent 057bedb65b
commit e10a865b05
12 changed files with 111 additions and 184 deletions

View File

@ -1,103 +0,0 @@
From 6b6a130924ba6d881d2db44d6382ab96fcb061d9 Mon Sep 17 00:00:00 2001
From: Daniel Micay <danielmicay@gmail.com>
Date: Tue, 3 Apr 2018 12:32:43 -0400
Subject: [PATCH] Add optional automated signing
Change-Id: I2a7efac25b3e48d3edcadcb9a57e9a7dbc7dfac0
---
core/Makefile | 43 +++++++++++++++++++++++++++++++++++++------
1 file changed, 37 insertions(+), 6 deletions(-)
diff --git a/core/Makefile b/core/Makefile
index 3fb424733..207088b93 100644
--- a/core/Makefile
+++ b/core/Makefile
@@ -416,6 +416,11 @@ ifneq ($(OTA_PACKAGE_SIGNING_KEY),)
DEFAULT_KEY_CERT_PAIR := $(OTA_PACKAGE_SIGNING_KEY)
endif
+ifneq ($(SIGNING_KEY_DIR),)
+ KEY_CERT_DIR := $(SIGNING_KEY_DIR)
+ DEFAULT_KEY_CERT_PAIR := $(SIGNING_KEY_DIR)/releasekey
+endif
+
# Rules that need to be present for the all targets, even
# if they don't do anything.
.PHONY: systemimage
@@ -1033,11 +1038,15 @@ endif
# substitute other keys for this one.
OTA_PUBLIC_KEYS := $(DEFAULT_SYSTEM_DEV_CERTIFICATE).x509.pem
-ifneq ($(OTA_PACKAGE_SIGNING_KEY),)
- OTA_PUBLIC_KEYS := $(OTA_PACKAGE_SIGNING_KEY).x509.pem
- PRODUCT_EXTRA_RECOVERY_KEYS := $(DEFAULT_SYSTEM_DEV_CERTIFICATE)
+ifneq ($(SIGNING_KEY_DIR),)
+ OTA_PUBLIC_KEYS := $(SIGNING_KEY_DIR)/releasekey.x509.pem
+ PRODUCT_EXTRA_RECOVERY_KEYS += $(SIGNING_KEY_DIR)/extra
+else
+ ifneq ($(OTA_PACKAGE_SIGNING_KEY),)
+ OTA_PUBLIC_KEYS := $(OTA_PACKAGE_SIGNING_KEY).x509.pem
+ PRODUCT_EXTRA_RECOVERY_KEYS := $(DEFAULT_SYSTEM_DEV_CERTIFICATE)
+ endif
endif
-
# Generate a file containing the keys that will be read by the
# recovery binary.
RECOVERY_INSTALL_OTA_KEYS := \
@@ -1823,6 +1832,13 @@ $(BUILT_TARGET_FILES_PACKAGE): intermediates := $(intermediates)
$(BUILT_TARGET_FILES_PACKAGE): \
zip_root := $(intermediates)/$(name)
+SIGNED_TARGET_FILES_PACKAGE := $(intermediates)/signed-$(name).zip
+MAYBE_SIGNED_TARGET_FILES_PACKAGE := $(BUILT_TARGET_FILES_PACKAGE)
+
+ifneq ($(SIGNING_KEY_DIR),)
+ MAYBE_SIGNED_TARGET_FILES_PACKAGE := $(SIGNED_TARGET_FILES_PACKAGE)
+endif
+
# $(1): Directory to copy
# $(2): Location to copy it to
# The "ls -A" is to prevent "acp s/* d" from failing if s is empty.
@@ -2190,6 +2206,12 @@ else
OTA_FROM_TARGET_SCRIPT := $(TARGET_RELEASETOOL_OTA_FROM_TARGET_SCRIPT)
endif
+ifeq ($(TARGET_RELEASETOOL_SIGN_TARGET_SCRIPT),)
+ SIGN_TARGET_SCRIPT := ./build/tools/releasetools/sign_target_files_apks
+else
+ SIGN_TARGET_SCRIPT := $(TARGET_RELEASETOOL_SIGN_TARGET_SCRIPT)
+endif
+
ifeq ($(WITH_GMS),true)
$(INTERNAL_OTA_PACKAGE_TARGET): backuptool := false
else
@@ -2210,7 +2232,16 @@ ifneq ($(BLOCK_BASED_OTA),false)
$(INTERNAL_OTA_PACKAGE_TARGET): block_based := --block
endif
-$(INTERNAL_OTA_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE)
+$(SIGNED_TARGET_FILES_PACKAGE): $(BUILT_TARGET_FILES_PACKAGE)
+ @echo "$(SIGN_TARGET_SCRIPT)" > $(PRODUCT_OUT)/sign_script_path
+ @echo -e ${CL_YLW}"Sign target files:"${CL_RST}" $@"
+ $(hide) $(SIGN_TARGET_SCRIPT) \
+ -d $(KEY_CERT_DIR) \
+ -o \
+ $(BUILT_TARGET_FILES_PACKAGE) \
+ $(SIGNED_TARGET_FILES_PACKAGE)
+
+$(INTERNAL_OTA_PACKAGE_TARGET): $(MAYBE_SIGNED_TARGET_FILES_PACKAGE)
@echo "$(OTA_FROM_TARGET_SCRIPT)" > $(PRODUCT_OUT)/ota_script_path
@echo "Package OTA: $@"
$(hide) PATH=$(foreach p,$(INTERNAL_USERIMAGES_BINARY_PATHS),$(p):)$$PATH MKBOOTIMG=$(MKBOOTIMG) \
@@ -2220,7 +2251,7 @@ $(INTERNAL_OTA_PACKAGE_TARGET): $(BUILT_TARGET_FILES_PACKAGE)
-k $(KEY_CERT_PAIR) \
--backup=$(backuptool) \
$(if $(OEM_OTA_CONFIG), -o $(OEM_OTA_CONFIG)) \
- $(BUILT_TARGET_FILES_PACKAGE) $@
+ $(MAYBE_SIGNED_TARGET_FILES_PACKAGE) $@
CM_TARGET_PACKAGE := $(PRODUCT_OUT)/lineage-$(LINEAGE_VERSION).zip
--
2.16.3

View File

@ -140,38 +140,73 @@ audit2allowADB() {
} }
export -f audit2allowADB; export -f audit2allowADB;
signRelease() { processRelease() {
#https://github.com/GrapheneOS/script/blob/pie/release.sh #https://github.com/GrapheneOS/script/blob/pie/release.sh
DEVICE=$1; DEVICE="$1";
VERITY=$2; BLOCK="$2";
VERITY="$3";
DATE=$(date -u '+%Y%m%d') DATE=$(date -u '+%Y%m%d')
KEY_DIR=$DOS_SIGNING_KEYS; KEY_DIR="$DOS_SIGNING_KEYS";
PREFIX="lineage_";
VERSION=$(echo $DOS_VERSION | cut -f2 -d "-"); VERSION=$(echo $DOS_VERSION | cut -f2 -d "-");
TARGET_FILES=divested-$VERSION-$DATE-dos-$DEVICE-target_files.zip; PREFIX="$DOS_BRANDING_ZIP_PREFIX-$VERSION-$DATE-dos-$DEVICE";
ARCHIVE="$DOS_BUILDS/$DOS_VERSION/release_keys/";
OUT_DIR="$DOS_BUILD_BASE/out/target/product/$DEVICE/";
#Arguments
if [ "$BLOCK" != false ]; then
BLOCK_SWITCHES="--block";
fi;
if [ "$VERITY" = true ]; then if [ "$VERITY" = true ]; then
VERITY_SWITCHES=(--replace_verity_public_key "$KEY_DIR/verity_key.pub" \ VERITY_SWITCHES=(--replace_verity_public_key "$KEY_DIR/verity_key.pub" \
--replace_verity_private_key "$KEY_DIR/verity" \ --replace_verity_private_key "$KEY_DIR/verity" \
--replace_verity_keyid "$KEY_DIR/verity.x509.pem"); --replace_verity_keyid "$KEY_DIR/verity.x509.pem");
fi; fi;
#Target Files
build/tools/releasetools/sign_target_files_apks -o -d "$KEY_DIR" \ build/tools/releasetools/sign_target_files_apks -o -d "$KEY_DIR" \
"${VERITY_SWITCHES[@]}" \ "${VERITY_SWITCHES[@]}" \
out/target/product/$DEVICE/obj/PACKAGING/target_files_intermediates/$PREFIX$DEVICE-target_files-*.zip \ $OUT_DIR/obj/PACKAGING/target_files_intermediates/*$DEVICE-target_files-*.zip \
$OUT/$TARGET_FILES; $OUT_DIR/$PREFIX-target_files.zip;
INCREMENTAL_ID=$(grep "ro.build.version.incremental" $OUT_DIR/system/build.prop | cut -f2 -d "=" | sed 's/\.//g');
echo $INCREMENTAL_ID > $OUT_DIR/$PREFIX-target_files.zip.id;
build/tools/releasetools/ota_from_target_files --block -k "$KEY_DIR/releasekey" \ #Image
$OUT/$TARGET_FILES \ #build/tools/releasetools/img_from_target_files $OUT_DIR/$PREFIX-target_files.zip \
$OUT/divested-$VERSION-$DATE-dos-$DEVICE-ota.zip; # $OUT_DIR/$PREFIX-img.zip || exit 1;
md5sum $OUT/divested-$VERSION-$DATE-dos-$DEVICE-ota.zip > $OUT/divested-$VERSION-$DATE-dos-$DEVICE-ota.zip.md5sum; #OTA
build/tools/releasetools/ota_from_target_files $BLOCK_SWITCHES -t 8 -k "$KEY_DIR/releasekey" \
$OUT_DIR/$PREFIX-target_files.zip \
$OUT_DIR/$PREFIX-ota.zip;
md5sum $OUT_DIR/$PREFIX-ota.zip > $OUT_DIR/$PREFIX-ota.zip.md5sum;
#build/tools/releasetools/img_from_target_files $OUT/$TARGET_FILES \ #Deltas
# $OUT/divested-$VERSION-$DATE-dos-$DEVICE-img.zip || exit 1; if [ "$DOS_GENERATE_DELTAS" = true ]; then
for LAST_TARGET_FILES in $ARCHIVE/target_files/$DOS_BRANDING_ZIP_PREFIX-$VERSION-*-dos-$DEVICE-target_files.zip; do
if [[ -f "$LAST_TARGET_FILES.id" ]]; then
LAST_INCREMENTAL_ID=$(cat "$LAST_TARGET_FILES.id");
build/tools/releasetools/ota_from_target_files $BLOCK_SWITCHES -t 8 -k "$KEY_DIR" -i \
"$LAST_TARGET_FILES" \
$OUT_DIR/$PREFIX-target_files.zip \
$OUT_DIR/$PREFIX-incremental_$LAST_INCREMENTAL_ID.zip;
md5sum $OUT_DIR/$PREFIX-incremental_$LAST_INCREMENTAL_ID.zip > $OUT_DIR/$PREFIX-incremental_$LAST_INCREMENTAL_ID.zip.md5sum;
fi;
done;
fi;
#Copy to archive
if [ "$DOS_AUTO_ARCHIVE_BUILDS" = true ]; then
mkdir -vp $ARCHIVE;
mkdir -vp $ARCHIVE/target_files;
mkdir -vp $ARCHIVE/incrementals;
cp -v $OUT_DIR/$PREFIX-target_files.* $ARCHIVE/target_files/;
cp -v $OUT_DIR/$PREFIX-ota.zip* $ARCHIVE/;
cp -v $OUT_DIR/$PREFIX-incremental_*.zip* $ARCHIVE/incrementals/;
fi;
} }
export -f signRelease; export -f processRelease;
disableDexPreOpt() { disableDexPreOpt() {
cd "$DOS_BUILD_BASE$1"; cd "$DOS_BUILD_BASE$1";

View File

@ -35,7 +35,7 @@ scanWorkspaceForMalware() {
export -f scanWorkspaceForMalware; export -f scanWorkspaceForMalware;
buildDevice() { buildDevice() {
brunch "cm_$1-user"; brunch "cm_$1-user" && processRelease $1 false false;
} }
export -f buildDevice; export -f buildDevice;
@ -48,7 +48,7 @@ export -f buildDeviceDebug;
buildAll() { buildAll() {
if [ "$DOS_MALWARE_SCAN_ENABLED" = true ]; then scanWorkspaceForMalware; fi; if [ "$DOS_MALWARE_SCAN_ENABLED" = true ]; then scanWorkspaceForMalware; fi;
if [ "$DOS_OPTIMIZE_IMAGES" = true ]; then optimizeImagesRecursive "$DOS_BUILD_BASE"; fi; if [ "$DOS_OPTIMIZE_IMAGES" = true ]; then optimizeImagesRecursive "$DOS_BUILD_BASE"; fi;
brunch lineage_nex-user; brunch lineage_nex-user && processRelease nex false false;
} }
export -f buildAll; export -f buildAll;

View File

@ -30,9 +30,6 @@
#buildDevice [device] #buildDevice [device]
#buildAll #buildAll
#Generate an incremental
#./build/tools/releasetools/ota_from_target_files --block -t 8 -i old.zip new.zip update.zip
#Generate firmware deblobber #Generate firmware deblobber
#mka firmware_deblobber #mka firmware_deblobber

View File

@ -35,10 +35,15 @@ scanWorkspaceForMalware() {
export -f scanWorkspaceForMalware; export -f scanWorkspaceForMalware;
buildDevice() { buildDevice() {
brunch "lineage_$1-user"; brunch "lineage_$1-user" && processRelease $1 true $2;
} }
export -f buildDevice; export -f buildDevice;
buildDeviceUserDebug() {
brunch "lineage_$1-userdebug" && processRelease $1 true $2;
}
export -f buildDeviceUserDebug;
buildDeviceDebug() { buildDeviceDebug() {
unset SIGNING_KEY_DIR; unset SIGNING_KEY_DIR;
brunch "lineage_$1-eng"; brunch "lineage_$1-eng";
@ -49,55 +54,55 @@ buildAll() {
if [ "$DOS_MALWARE_SCAN_ENABLED" = true ]; then scanWorkspaceForMalware; fi; if [ "$DOS_MALWARE_SCAN_ENABLED" = true ]; then scanWorkspaceForMalware; fi;
if [ "$DOS_OPTIMIZE_IMAGES" = true ]; then optimizeImagesRecursive "$DOS_BUILD_BASE"; fi; if [ "$DOS_OPTIMIZE_IMAGES" = true ]; then optimizeImagesRecursive "$DOS_BUILD_BASE"; fi;
#Select devices are userdebug due to SELinux policy issues #Select devices are userdebug due to SELinux policy issues
brunch lineage_clark-user; buildDevice clark;
brunch lineage_thor-userdebug; buildDeviceUserDebug thor;
brunch lineage_grouper-user; #needs manual patching - one-repo vendor blob patch buildDevice grouper; #needs manual patching - one-repo vendor blob patch
brunch lineage_h815-user; buildDevice h815;
brunch lineage_herolte-user; buildDevice herolte;
brunch lineage_himaul-user; buildDevice himaul;
brunch lineage_i9100-userdebug; buildDeviceUserDebug i9100;
brunch lineage_i9300-userdebug; buildDeviceUserDebug i9300;
brunch lineage_i9305-user; buildDevice i9305;
brunch lineage_maguro-user; buildDevice maguro;
brunch lineage_manta-user; buildDevice manta;
brunch lineage_n5110-user; buildDevice n5110;
#brunch lineage_n7100-user; #device/samsung/n7100/selinux/device.te:5:ERROR 'duplicate declaration of type/attribute' at token ';': type hpd_device, dev_type; type mfc_device, dev_type; buildDevice n7100; #device/samsung/n7100/selinux/device.te:5:ERROR 'duplicate declaration of type/attribute' at token ';': type hpd_device, dev_type; type mfc_device, dev_type;
brunch lineage_osprey-user; buildDevice osprey;
brunch lineage_toro-user; buildDevice toro;
brunch lineage_toroplus-user; buildDevice toroplus;
brunch lineage_Z00T-user; buildDevice Z00T;
#The following are all superseded, and should only be enabled if the newer version is broken (not building/booting/etc.) #The following are all superseded, and should only be enabled if the newer version is broken (not building/booting/etc.)
if [ "$DOS_BUILDALL_SUPERSEDED" = true ]; then if [ "$DOS_BUILDALL_SUPERSEDED" = true ]; then
brunch lineage_angler-user; buildDevice angler;
brunch lineage_axon7-user; buildDevice axon7;
brunch lineage_bullhead-user; buildDevice bullhead;
brunch lineage_bacon-user; buildDevice bacon;
brunch lineage_crackling-user; buildDevice crackling;
brunch lineage_d802-user; buildDevice d802;
brunch lineage_d852-user; buildDevice d852;
brunch lineage_d855-user; buildDevice d855;
brunch lineage_dragon-user; buildDevice dragon;
brunch lineage_ether-user; buildDevice ether;
brunch lineage_flo-user; buildDevice flo;
brunch lineage_flounder-user; buildDevice flounder;
brunch lineage_FP2-user; buildDevice FP2;
brunch lineage_fugu-user; buildDevice fugu;
brunch lineage_griffin-user; buildDevice griffin;
brunch lineage_h850-user; buildDevice h850;
brunch lineage_ham-user; buildDevice ham;
brunch lineage_hammerhead-user; buildDevice hammerhead;
brunch lineage_jfltexx-user; #broken - drivers/video/msm/mdp.c:401:1: warning: the frame size of 1032 bytes is larger than 1024 bytes [-Wframe-larger-than=] buildDevice jfltexx; #broken - drivers/video/msm/mdp.c:401:1: warning: the frame size of 1032 bytes is larger than 1024 bytes [-Wframe-larger-than=]
brunch lineage_kipper-user; buildDevice kipper;
brunch lineage_klte-user; buildDevice klte;
brunch lineage_m8-user; buildDevice m8;
brunch lineage_mako-user; buildDevice mako;
brunch lineage_marlin-user; buildDevice marlin;
brunch lineage_sailfish-user; buildDevice sailfish;
brunch lineage_shamu-user; buildDevice shamu;
brunch lineage_us996-user; buildDevice us996;
brunch lineage_us997-user; buildDevice us997;
brunch lineage_victara-user; #needs manual patching - fwb xml: fused: dangling tag buildDevice victara; #needs manual patching - fwb xml: fused: dangling tag
fi; fi;
} }
export -f buildAll; export -f buildAll;

View File

@ -30,9 +30,6 @@
#buildDevice [device] #buildDevice [device]
#buildAll #buildAll
#Generate an incremental
#./build/tools/releasetools/ota_from_target_files --block -t 8 -i old.zip new.zip update.zip
#Generate firmware deblobber #Generate firmware deblobber
#mka firmware_deblobber #mka firmware_deblobber
@ -69,7 +66,6 @@ enterAndClear "bootable/recovery";
patch -p1 < "$DOS_PATCHES/android_bootable_recovery/0001-Squash_Menus.patch"; #What's a back button? patch -p1 < "$DOS_PATCHES/android_bootable_recovery/0001-Squash_Menus.patch"; #What's a back button?
enterAndClear "build"; enterAndClear "build";
patch -p1 < "$DOS_PATCHES/android_build/0001-Automated_Build_Signing.patch"; #Automated build signing (CopperheadOS-13.0)
sed -i '50i$(my_res_package): PRIVATE_AAPT_FLAGS += --auto-add-overlay' core/aapt2.mk; sed -i '50i$(my_res_package): PRIVATE_AAPT_FLAGS += --auto-add-overlay' core/aapt2.mk;
sed -i '296iLOCAL_AAPT_FLAGS += --auto-add-overlay' core/package_internal.mk; sed -i '296iLOCAL_AAPT_FLAGS += --auto-add-overlay' core/package_internal.mk;

View File

@ -35,7 +35,7 @@ scanWorkspaceForMalware() {
export -f scanWorkspaceForMalware; export -f scanWorkspaceForMalware;
buildDevice() { buildDevice() {
brunch "lineage_$1-user" && signRelease $1 $2; brunch "lineage_$1-user" && processRelease $1 true $2;
} }
export -f buildDevice; export -f buildDevice;

View File

@ -30,9 +30,6 @@
#buildDevice [device] #buildDevice [device]
#buildAll #buildAll
#Generate an incremental
#./build/tools/releasetools/ota_from_target_files --block -t 8 -i old.zip new.zip update.zip
#Generate firmware deblobber #Generate firmware deblobber
#mka firmware_deblobber #mka firmware_deblobber

View File

@ -35,7 +35,7 @@ scanWorkspaceForMalware() {
export -f scanWorkspaceForMalware; export -f scanWorkspaceForMalware;
buildDevice() { buildDevice() {
brunch "lineage_$1-user" && signRelease $1 $2; brunch "lineage_$1-user" && processRelease $1 true $2;
} }
export -f buildDevice; export -f buildDevice;

View File

@ -30,9 +30,6 @@
#buildDevice [device] #buildDevice [device]
#buildAll #buildAll
#Generate an incremental
#./build/tools/releasetools/ota_from_target_files --block -t 8 -i old.zip new.zip update.zip
#Generate firmware deblobber #Generate firmware deblobber
#mka firmware_deblobber #mka firmware_deblobber

View File

@ -53,6 +53,9 @@ export DOS_DEFAULT_DNS_PRESET="OpenNIC"; #Sets default DNS. Options: CensurfriDN
export DOS_GPS_NTP_SERVER="1.android.pool.ntp.org"; #Options: Any NTP pool export DOS_GPS_NTP_SERVER="1.android.pool.ntp.org"; #Options: Any NTP pool
export DOS_GPS_SUPL_HOST="supl.google.com"; #Options: supl.{google,vodafone,sonyericsson}.com export DOS_GPS_SUPL_HOST="supl.google.com"; #Options: supl.{google,vodafone,sonyericsson}.com
export DOS_AUTO_ARCHIVE_BUILDS=true; #Copies files to $DOS_BUILDS after signing
export DOS_GENERATE_DELTAS=true; #Creates deltas from existing target_files in $DOS_BUILDS
export DOS_BRANDING_NAME="DivestOS"; export DOS_BRANDING_NAME="DivestOS";
export DOS_BRANDING_ZIP_PREFIX="divested"; export DOS_BRANDING_ZIP_PREFIX="divested";
export DOS_BRANDING_BOOTANIMATION_FONT="Fira-Sans-Bold"; #Options: $ convert -list font export DOS_BRANDING_BOOTANIMATION_FONT="Fira-Sans-Bold"; #Options: $ convert -list font
@ -111,6 +114,7 @@ export DOS_TMP_DIR="/tmp/dos_tmp";
mkdir -p "$DOS_TMP_DIR"; mkdir -p "$DOS_TMP_DIR";
export DOS_HOSTS_FILE="$DOS_TMP_DIR/hosts"; export DOS_HOSTS_FILE="$DOS_TMP_DIR/hosts";
export DOS_BUILDS=$DOS_WORKSPACE_ROOT"Builds/";
export DOS_PREBUILT_APPS=$DOS_WORKSPACE_ROOT"PrebuiltApps/"; export DOS_PREBUILT_APPS=$DOS_WORKSPACE_ROOT"PrebuiltApps/";
export DOS_PATCHES_COMMON=$DOS_WORKSPACE_ROOT"Patches/Common/"; export DOS_PATCHES_COMMON=$DOS_WORKSPACE_ROOT"Patches/Common/";
export DOS_PATCHES=$DOS_WORKSPACE_ROOT"Patches/$BUILD_WORKING_DIR/"; export DOS_PATCHES=$DOS_WORKSPACE_ROOT"Patches/$BUILD_WORKING_DIR/";

3
TODO
View File

@ -1,8 +1,7 @@
Last updated: 2019-02-08 Last updated: 2019-08-29
High Priority (Release blockers) High Priority (Release blockers)
Build Build
- Automate incremental update generation
- New signing keys - New signing keys
Project Project
- GPG all the things! - GPG all the things!